diff --git a/pir/pir-impl/build.gradle b/pir/pir-impl/build.gradle index 0f1ce5c8fbcb..81c37935703e 100644 --- a/pir/pir-impl/build.gradle +++ b/pir/pir-impl/build.gradle @@ -74,6 +74,7 @@ dependencies { testImplementation project(path: ':feature-toggles-test') testImplementation CashApp.turbine testImplementation AndroidX.test.ext.junit + testImplementation AndroidX.room.testing testImplementation Testing.robolectric testImplementation "androidx.lifecycle:lifecycle-runtime-testing:_" testImplementation AndroidX.work.testing @@ -98,6 +99,9 @@ dependencies { android { namespace "com.duckduckgo.pir.impl" + sourceSets { + test.assets.srcDirs += files("$projectDir/schemas") + } anvil { generateDaggerFactories = true // default is false } diff --git a/pir/pir-impl/schemas/com.duckduckgo.pir.impl.store.PirDatabase/17.json b/pir/pir-impl/schemas/com.duckduckgo.pir.impl.store.PirDatabase/17.json new file mode 100644 index 000000000000..e0c4e5884182 --- /dev/null +++ b/pir/pir-impl/schemas/com.duckduckgo.pir.impl.store.PirDatabase/17.json @@ -0,0 +1,946 @@ +{ + "formatVersion": 1, + "database": { + "version": 17, + "identityHash": "642943f91e5ef7438c568716d5a05864", + "entities": [ + { + "tableName": "pir_broker_json_etag", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`fileName` TEXT NOT NULL, `etag` TEXT NOT NULL, PRIMARY KEY(`fileName`))", + "fields": [ + { + "fieldPath": "fileName", + "columnName": "fileName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "etag", + "columnName": "etag", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "fileName" + ] + } + }, + { + "tableName": "pir_broker_details", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`name` TEXT NOT NULL, `fileName` TEXT NOT NULL, `url` TEXT NOT NULL, `version` TEXT NOT NULL, `parent` TEXT, `addedDatetime` INTEGER NOT NULL, `removedAt` INTEGER NOT NULL, PRIMARY KEY(`name`))", + "fields": [ + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "fileName", + "columnName": "fileName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "url", + "columnName": "url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "version", + "columnName": "version", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "parent", + "columnName": "parent", + "affinity": "TEXT" + }, + { + "fieldPath": "addedDatetime", + "columnName": "addedDatetime", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "removedAt", + "columnName": "removedAt", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "name" + ] + } + }, + { + "tableName": "pir_broker_opt_out", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`brokerName` TEXT NOT NULL, `stepsJson` TEXT NOT NULL, `optOutUrl` TEXT, PRIMARY KEY(`brokerName`), FOREIGN KEY(`brokerName`) REFERENCES `pir_broker_details`(`name`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "stepsJson", + "columnName": "stepsJson", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "optOutUrl", + "columnName": "optOutUrl", + "affinity": "TEXT" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "brokerName" + ] + }, + "foreignKeys": [ + { + "table": "pir_broker_details", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "brokerName" + ], + "referencedColumns": [ + "name" + ] + } + ] + }, + { + "tableName": "pir_broker_scan", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`brokerName` TEXT NOT NULL, `stepsJson` TEXT, PRIMARY KEY(`brokerName`), FOREIGN KEY(`brokerName`) REFERENCES `pir_broker_details`(`name`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "stepsJson", + "columnName": "stepsJson", + "affinity": "TEXT" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "brokerName" + ] + }, + "foreignKeys": [ + { + "table": "pir_broker_details", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "brokerName" + ], + "referencedColumns": [ + "name" + ] + } + ] + }, + { + "tableName": "pir_broker_scheduling_config", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`brokerName` TEXT NOT NULL, `retryError` INTEGER NOT NULL, `confirmOptOutScan` INTEGER NOT NULL, `maintenanceScan` INTEGER NOT NULL, `maxAttempts` INTEGER, PRIMARY KEY(`brokerName`), FOREIGN KEY(`brokerName`) REFERENCES `pir_broker_details`(`name`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "retryError", + "columnName": "retryError", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "confirmOptOutScan", + "columnName": "confirmOptOutScan", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "maintenanceScan", + "columnName": "maintenanceScan", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "maxAttempts", + "columnName": "maxAttempts", + "affinity": "INTEGER" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "brokerName" + ] + }, + "foreignKeys": [ + { + "table": "pir_broker_details", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "brokerName" + ], + "referencedColumns": [ + "name" + ] + } + ] + }, + { + "tableName": "pir_user_profile", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `birthYear` INTEGER NOT NULL, `phone` TEXT, `deprecated` INTEGER NOT NULL, `user_firstName` TEXT NOT NULL, `user_lastName` TEXT NOT NULL, `user_middleName` TEXT, `user_suffix` TEXT, `address_city` TEXT NOT NULL, `address_state` TEXT NOT NULL, `address_street` TEXT, `address_zip` TEXT)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "birthYear", + "columnName": "birthYear", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "phone", + "columnName": "phone", + "affinity": "TEXT" + }, + { + "fieldPath": "deprecated", + "columnName": "deprecated", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "userName.firstName", + "columnName": "user_firstName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "userName.lastName", + "columnName": "user_lastName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "userName.middleName", + "columnName": "user_middleName", + "affinity": "TEXT" + }, + { + "fieldPath": "userName.suffix", + "columnName": "user_suffix", + "affinity": "TEXT" + }, + { + "fieldPath": "addresses.city", + "columnName": "address_city", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "addresses.state", + "columnName": "address_state", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "addresses.street", + "columnName": "address_street", + "affinity": "TEXT" + }, + { + "fieldPath": "addresses.zip", + "columnName": "address_zip", + "affinity": "TEXT" + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "pir_events_log", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`eventTimeInMillis` INTEGER NOT NULL, `eventType` TEXT NOT NULL, PRIMARY KEY(`eventTimeInMillis`))", + "fields": [ + { + "fieldPath": "eventTimeInMillis", + "columnName": "eventTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "eventType", + "columnName": "eventType", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "eventTimeInMillis" + ] + } + }, + { + "tableName": "pir_broker_scan_log", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`eventTimeInMillis` INTEGER NOT NULL, `brokerName` TEXT NOT NULL, `eventType` TEXT NOT NULL, PRIMARY KEY(`eventTimeInMillis`))", + "fields": [ + { + "fieldPath": "eventTimeInMillis", + "columnName": "eventTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "eventType", + "columnName": "eventType", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "eventTimeInMillis" + ] + } + }, + { + "tableName": "pir_scan_complete_brokers", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`brokerName` TEXT NOT NULL, `profileQueryId` INTEGER NOT NULL, `startTimeInMillis` INTEGER NOT NULL, `endTimeInMillis` INTEGER NOT NULL, `isSuccess` INTEGER NOT NULL, PRIMARY KEY(`brokerName`, `profileQueryId`))", + "fields": [ + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "profileQueryId", + "columnName": "profileQueryId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "startTimeInMillis", + "columnName": "startTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "endTimeInMillis", + "columnName": "endTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isSuccess", + "columnName": "isSuccess", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "brokerName", + "profileQueryId" + ] + } + }, + { + "tableName": "pir_opt_out_complete_brokers", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `brokerName` TEXT NOT NULL, `extractedProfile` TEXT NOT NULL, `startTimeInMillis` INTEGER NOT NULL, `endTimeInMillis` INTEGER NOT NULL, `isSubmitSuccess` INTEGER NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "extractedProfile", + "columnName": "extractedProfile", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "startTimeInMillis", + "columnName": "startTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "endTimeInMillis", + "columnName": "endTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isSubmitSuccess", + "columnName": "isSubmitSuccess", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "pir_opt_out_action_log", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `brokerName` TEXT NOT NULL, `extractedProfile` TEXT NOT NULL, `completionTimeInMillis` INTEGER NOT NULL, `actionType` TEXT NOT NULL, `isError` INTEGER NOT NULL, `result` TEXT NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "extractedProfile", + "columnName": "extractedProfile", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "completionTimeInMillis", + "columnName": "completionTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "actionType", + "columnName": "actionType", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "isError", + "columnName": "isError", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "result", + "columnName": "result", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "pir_extracted_profiles", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `profileQueryId` INTEGER NOT NULL, `brokerName` TEXT NOT NULL, `name` TEXT NOT NULL, `alternativeNames` TEXT NOT NULL, `age` TEXT NOT NULL, `addresses` TEXT NOT NULL, `phoneNumbers` TEXT NOT NULL, `relatives` TEXT NOT NULL, `profileUrl` TEXT NOT NULL, `identifier` TEXT NOT NULL, `reportId` TEXT NOT NULL, `email` TEXT NOT NULL, `fullName` TEXT NOT NULL, `dateAddedInMillis` INTEGER NOT NULL, `deprecated` INTEGER NOT NULL, `extras` TEXT NOT NULL DEFAULT '{}')", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "profileQueryId", + "columnName": "profileQueryId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "alternativeNames", + "columnName": "alternativeNames", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "age", + "columnName": "age", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "addresses", + "columnName": "addresses", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "phoneNumbers", + "columnName": "phoneNumbers", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "relatives", + "columnName": "relatives", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "profileUrl", + "columnName": "profileUrl", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "identifier", + "columnName": "identifier", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "reportId", + "columnName": "reportId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "email", + "columnName": "email", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "fullName", + "columnName": "fullName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "dateAddedInMillis", + "columnName": "dateAddedInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "deprecated", + "columnName": "deprecated", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "extras", + "columnName": "extras", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "'{}'" + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [ + { + "name": "index_pir_extracted_profiles_profileQueryId_brokerName_name_profileUrl_identifier", + "unique": true, + "columnNames": [ + "profileQueryId", + "brokerName", + "name", + "profileUrl", + "identifier" + ], + "orders": [], + "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_pir_extracted_profiles_profileQueryId_brokerName_name_profileUrl_identifier` ON `${TABLE_NAME}` (`profileQueryId`, `brokerName`, `name`, `profileUrl`, `identifier`)" + } + ] + }, + { + "tableName": "pir_scan_job_record", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`brokerName` TEXT NOT NULL, `userProfileId` INTEGER NOT NULL, `status` TEXT NOT NULL, `lastScanDateInMillis` INTEGER, `deprecated` INTEGER NOT NULL, `dateCreatedInMillis` INTEGER NOT NULL, PRIMARY KEY(`brokerName`, `userProfileId`))", + "fields": [ + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "userProfileId", + "columnName": "userProfileId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "status", + "columnName": "status", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "lastScanDateInMillis", + "columnName": "lastScanDateInMillis", + "affinity": "INTEGER" + }, + { + "fieldPath": "deprecated", + "columnName": "deprecated", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "dateCreatedInMillis", + "columnName": "dateCreatedInMillis", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "brokerName", + "userProfileId" + ] + } + }, + { + "tableName": "pir_optout_job_record", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`extractedProfileId` INTEGER NOT NULL, `brokerName` TEXT NOT NULL, `userProfileId` INTEGER NOT NULL, `status` TEXT NOT NULL, `attemptCount` INTEGER NOT NULL, `lastOptOutAttemptDate` INTEGER, `optOutRequestedDate` INTEGER NOT NULL, `optOutFormSubmittedDate` INTEGER, `optOutRemovedDate` INTEGER NOT NULL, `deprecated` INTEGER NOT NULL, `dateCreatedInMillis` INTEGER NOT NULL, `reporting_sevenDayConfirmationReportSentDateMs` INTEGER NOT NULL, `reporting_fourteenDayConfirmationReportSentDateMs` INTEGER NOT NULL, `reporting_twentyOneDayConfirmationReportSentDateMs` INTEGER NOT NULL, `reporting_fortyTwoDayConfirmationReportSentDateMs` INTEGER NOT NULL, PRIMARY KEY(`extractedProfileId`))", + "fields": [ + { + "fieldPath": "extractedProfileId", + "columnName": "extractedProfileId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "userProfileId", + "columnName": "userProfileId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "status", + "columnName": "status", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "attemptCount", + "columnName": "attemptCount", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastOptOutAttemptDate", + "columnName": "lastOptOutAttemptDate", + "affinity": "INTEGER" + }, + { + "fieldPath": "optOutRequestedDate", + "columnName": "optOutRequestedDate", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "optOutFormSubmittedDate", + "columnName": "optOutFormSubmittedDate", + "affinity": "INTEGER" + }, + { + "fieldPath": "optOutRemovedDate", + "columnName": "optOutRemovedDate", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "deprecated", + "columnName": "deprecated", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "dateCreatedInMillis", + "columnName": "dateCreatedInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reporting.sevenDayConfirmationReportSentDateMs", + "columnName": "reporting_sevenDayConfirmationReportSentDateMs", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reporting.fourteenDayConfirmationReportSentDateMs", + "columnName": "reporting_fourteenDayConfirmationReportSentDateMs", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reporting.twentyOneDayConfirmationReportSentDateMs", + "columnName": "reporting_twentyOneDayConfirmationReportSentDateMs", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reporting.fortyTwoDayConfirmationReportSentDateMs", + "columnName": "reporting_fortyTwoDayConfirmationReportSentDateMs", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "extractedProfileId" + ] + } + }, + { + "tableName": "pir_broker_mirror_sites", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`name` TEXT NOT NULL, `url` TEXT NOT NULL, `addedAt` INTEGER NOT NULL, `removedAt` INTEGER NOT NULL, `optOutUrl` TEXT NOT NULL, `parentSite` TEXT NOT NULL, PRIMARY KEY(`name`))", + "fields": [ + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "url", + "columnName": "url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "addedAt", + "columnName": "addedAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "removedAt", + "columnName": "removedAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "optOutUrl", + "columnName": "optOutUrl", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "parentSite", + "columnName": "parentSite", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "name" + ] + } + }, + { + "tableName": "pir_email_confirmation_job_record", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`extractedProfileId` INTEGER NOT NULL, `brokerName` TEXT NOT NULL, `userProfileId` INTEGER NOT NULL, `email` TEXT NOT NULL, `attemptId` TEXT NOT NULL, `dateCreatedInMillis` INTEGER NOT NULL, `emailConfirmationLink` TEXT NOT NULL, `linkFetchAttemptCount` INTEGER NOT NULL, `lastLinkFetchDateInMillis` INTEGER NOT NULL, `jobAttemptCount` INTEGER NOT NULL, `lastJobAttemptDateInMillis` INTEGER NOT NULL, `lastJobAttemptActionId` TEXT NOT NULL, `deprecated` INTEGER NOT NULL, PRIMARY KEY(`extractedProfileId`))", + "fields": [ + { + "fieldPath": "extractedProfileId", + "columnName": "extractedProfileId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "userProfileId", + "columnName": "userProfileId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "email", + "columnName": "email", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "attemptId", + "columnName": "attemptId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "dateCreatedInMillis", + "columnName": "dateCreatedInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "emailConfirmationLink", + "columnName": "emailConfirmationLink", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "linkFetchAttemptCount", + "columnName": "linkFetchAttemptCount", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastLinkFetchDateInMillis", + "columnName": "lastLinkFetchDateInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "jobAttemptCount", + "columnName": "jobAttemptCount", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastJobAttemptDateInMillis", + "columnName": "lastJobAttemptDateInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastJobAttemptActionId", + "columnName": "lastJobAttemptActionId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "deprecated", + "columnName": "deprecated", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "extractedProfileId" + ] + } + }, + { + "tableName": "pir_email_confirmation_log", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`eventTimeInMillis` INTEGER NOT NULL, `eventType` TEXT NOT NULL, `value` TEXT NOT NULL, PRIMARY KEY(`eventTimeInMillis`))", + "fields": [ + { + "fieldPath": "eventTimeInMillis", + "columnName": "eventTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "eventType", + "columnName": "eventType", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "value", + "columnName": "value", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "eventTimeInMillis" + ] + } + } + ], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '642943f91e5ef7438c568716d5a05864')" + ] + } +} \ No newline at end of file diff --git a/pir/pir-impl/src/androidTest/java/com/duckduckgo/pir/impl/integration/PirEndToEndTest.kt b/pir/pir-impl/src/androidTest/java/com/duckduckgo/pir/impl/integration/PirEndToEndTest.kt index 0bb8725f60c7..1318c56f28ed 100644 --- a/pir/pir-impl/src/androidTest/java/com/duckduckgo/pir/impl/integration/PirEndToEndTest.kt +++ b/pir/pir-impl/src/androidTest/java/com/duckduckgo/pir/impl/integration/PirEndToEndTest.kt @@ -492,6 +492,16 @@ class PirEndToEndTest { val removedBrokerProfiles = extractedProfiles.filter { it.brokerName == removedBrokerName } assertTrue("Removed broker should not have extracted profiles", removedBrokerProfiles.isEmpty()) + // Verify extras survive the real encrypted database at both the profile and address level + extractedProfiles.forEach { profile -> + assertEquals("Profile extras should be stored", mapOf("middleInitial" to "M"), profile.extras) + assertEquals( + "Address extras should be stored", + mapOf("county" to "Kings"), + profile.addresses.single().extras, + ) + } + println("===================== STEP 4: Verify Opt-Out Jobs Created ====================") // Verify that opt-out jobs were created for active broker only @@ -655,6 +665,78 @@ class PirEndToEndTest { ) } + @Test + fun testRescanRefreshesStoredProfileInPlace() = runBlocking { + println("==================== STEP 1: Initial scan ====================") + + loadBrokers() + pirRepository.replaceUserProfile(testProfile) + + val initialResult = pirJobsRunner.runEligibleJobs(context, PirExecutionType.MANUAL_INITIAL) + assertTrue("Initial scan should succeed", initialResult.isSuccess) + + val initialProfile = pirRepository.getAllExtractedProfiles().single() + assertEquals("Profile extras should be stored", mapOf("middleInitial" to "M"), initialProfile.extras) + + val optOutJobRecord = pirSchedulingRepository.getAllValidOptOutJobRecords().single() + assertEquals( + "Opt-out job should point at the stored profile", + initialProfile.dbId, + optOutJobRecord.extractedProfileId, + ) + + println("==================== STEP 2: Re-scan with new extras ====================") + + // Simulate a broker config that now extracts an extra field + fakePirMessagingInterface.setExtractExtras( + profileExtras = mapOf("middleInitial" to "M", "county" to "Cook"), + addressExtras = mapOf("county" to "Cook", "zip" to "60601"), + ) + fakePirMessagingInterface.clearPushedActions() + fakeTimeProvider.advanceByHours(73) + + val rescanResult = pirJobsRunner.runEligibleJobs(context, PirExecutionType.SCHEDULED) + assertTrue("Re-scan should succeed", rescanResult.isSuccess) + + println("==================== STEP 3: Verify row refreshed, not duplicated ====================") + + val refreshedProfile = pirRepository.getAllExtractedProfiles().single() + // Same row: an accidental delete-and-reinsert would change the autogenerated id and orphan the job records + assertEquals("Row id should be preserved", initialProfile.dbId, refreshedProfile.dbId) + assertEquals( + "Date first found should be preserved", + initialProfile.dateAddedInMillis, + refreshedProfile.dateAddedInMillis, + ) + assertEquals( + "Profile extras should be refreshed", + mapOf("middleInitial" to "M", "county" to "Cook"), + refreshedProfile.extras, + ) + assertEquals( + "Address extras should be refreshed", + mapOf("county" to "Cook", "zip" to "60601"), + refreshedProfile.addresses.single().extras, + ) + + // The still-listed profile must not have been treated as removed + val refreshedOptOutJob = pirSchedulingRepository.getAllValidOptOutJobRecords().single() + assertEquals( + "Opt-out job should still resolve to the same profile", + refreshedProfile.dbId, + refreshedOptOutJob.extractedProfileId, + ) + assertFalse( + "Opt-out job should not be marked REMOVED", + refreshedOptOutJob.status == OptOutJobStatus.REMOVED, + ) + assertEquals( + "Extracted profile should still be resolvable by id", + refreshedProfile, + pirRepository.getExtractedProfile(refreshedProfile.dbId), + ) + } + @Test fun testScanGracefullyHandlesUnknownActions() = runBlocking { println("==================== STEP 1: Setup broker with unknown scan action ====================") diff --git a/pir/pir-impl/src/androidTest/java/com/duckduckgo/pir/impl/integration/fakes/FakePirMessagingInterface.kt b/pir/pir-impl/src/androidTest/java/com/duckduckgo/pir/impl/integration/fakes/FakePirMessagingInterface.kt index c6f37f569c16..c99775bee030 100644 --- a/pir/pir-impl/src/androidTest/java/com/duckduckgo/pir/impl/integration/fakes/FakePirMessagingInterface.kt +++ b/pir/pir-impl/src/androidTest/java/com/duckduckgo/pir/impl/integration/fakes/FakePirMessagingInterface.kt @@ -49,6 +49,9 @@ class FakePirMessagingInterface(moshi: Moshi) : JsMessaging { // Configurable extract response for confirmation scan testing private var nextExtractResponseEmpty: Boolean = false + private var profileExtras: Map = mapOf("middleInitial" to "M") + private var addressExtras: Map = mapOf("county" to "Kings") + /** * Configures the next Extract action to return an empty list of profiles. * Used to simulate confirmation scan where the profile has been removed. @@ -57,6 +60,18 @@ class FakePirMessagingInterface(moshi: Moshi) : JsMessaging { nextExtractResponseEmpty = empty } + /** + * Configures the extras returned by subsequent Extract actions, simulating a broker config that + * started extracting new fields. + */ + fun setExtractExtras( + profileExtras: Map, + addressExtras: Map, + ) { + this.profileExtras = profileExtras + this.addressExtras = addressExtras + } + fun clearPushedActions() { pushedActions.clear() } @@ -138,9 +153,11 @@ class FakePirMessagingInterface(moshi: Moshi) : JsMessaging { { "city": "New York", "state": "NY", - "fullAddress": "123 Main St, New York, NY 10001" + "fullAddress": "123 Main St, New York, NY 10001", + "extras": ${JSONObject(addressExtras)} } - ] + ], + "extras": ${JSONObject(profileExtras)} } ] """ diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirRunStateHandler.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirRunStateHandler.kt index e5b326eb280d..347656b3a7f5 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirRunStateHandler.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirRunStateHandler.kt @@ -668,6 +668,7 @@ class RealPirRunStateHandler @Inject constructor( city = item.city, state = item.state, fullAddress = item.fullAddress, + extras = item.extras.toStringExtras(), ) }, phoneNumbers = it.phoneNumbers, @@ -676,6 +677,7 @@ class RealPirRunStateHandler @Inject constructor( reportId = it.reportId.orEmpty(), email = it.email.orEmpty(), fullName = it.fullName.orEmpty(), + extras = it.extras.toStringExtras(), ) }.also { /** @@ -829,3 +831,10 @@ class RealPirRunStateHandler @Inject constructor( private const val KEY_STEPTYPE_OPTOUT = "optOut" } } + +/** + * The contract for extras is string values only; anything else is undefined behaviour, so entries we + * can't read as a string are dropped rather than coerced or allowed to fail the whole response. + */ +private fun Map?.toStringExtras(): Map = + this?.mapNotNull { (key, value) -> (value as? String)?.let { key to it } }?.toMap().orEmpty() diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirUtils.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirUtils.kt index c627ac9ada26..0d49b48e831e 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirUtils.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirUtils.kt @@ -16,7 +16,9 @@ package com.duckduckgo.pir.impl.common +import com.duckduckgo.pir.impl.models.AddressCityState import com.duckduckgo.pir.impl.models.ExtractedProfile +import com.duckduckgo.pir.impl.scripts.models.AddressCityStateParams import com.duckduckgo.pir.impl.scripts.models.ExtractedProfileParams internal fun List.splitIntoParts(parts: Int): List> { @@ -47,6 +49,22 @@ internal fun ExtractedProfile.toParams(fullName: String): ExtractedProfileParams profileUrl = this.profileUrl.ifEmpty { null }, fullName = fullName.ifEmpty { null }, email = this.email.ifEmpty { null }, + age = this.age.ifEmpty { null }, + addresses = + this.addresses + .map { + AddressCityStateParams( + city = it.city, + state = it.state, + fullAddress = it.fullAddress, + extras = it.extras.ifEmpty { null }, + ) + }.ifEmpty { null }, + phoneNumbers = this.phoneNumbers.ifEmpty { null }, + relatives = this.relatives.ifEmpty { null }, + alternativeNames = this.alternativeNames.ifEmpty { null }, + identifier = this.identifier.ifEmpty { null }, + extras = this.extras.ifEmpty { null }, ) } @@ -60,9 +78,12 @@ internal fun ExtractedProfile.matches(extractedProfile: ExtractedProfile): Boole return this.name == extractedProfile.name && this.age == extractedProfile.age && this.alternativeNames.isASubSetOrSuperSetOf(extractedProfile.alternativeNames) && this.relatives.isASubSetOrSuperSetOf(extractedProfile.relatives) && - this.addresses.isASubSetOrSuperSetOf(extractedProfile.addresses) + // extras are an open bag driven by broker config; including them would break mirror-site dedup whenever a new field ships + this.addresses.withoutExtras().isASubSetOrSuperSetOf(extractedProfile.addresses.withoutExtras()) } +private fun List.withoutExtras(): List = map { it.copy(extras = emptyMap()) } + private fun List.isASubSetOrSuperSetOf(other: List): Boolean { return this.containsAll(other) || other.containsAll(this) } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/models/ExtractedProfile.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/models/ExtractedProfile.kt index ea8876bbbf29..499fbb6a3c1e 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/models/ExtractedProfile.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/models/ExtractedProfile.kt @@ -42,10 +42,12 @@ data class ExtractedProfile( val identifier: String = "", val dateAddedInMillis: Long = 0L, val deprecated: Boolean = false, + val extras: Map = emptyMap(), ) data class AddressCityState( val city: String, val state: String, val fullAddress: String? = null, + val extras: Map = emptyMap(), ) diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scheduling/JobRecordUpdater.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scheduling/JobRecordUpdater.kt index b2e5ba24b325..ec2c90247893 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scheduling/JobRecordUpdater.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scheduling/JobRecordUpdater.kt @@ -19,7 +19,6 @@ package com.duckduckgo.pir.impl.scheduling import com.duckduckgo.common.utils.CurrentTimeProvider import com.duckduckgo.common.utils.DispatcherProvider import com.duckduckgo.di.scopes.AppScope -import com.duckduckgo.pir.impl.models.AddressCityState import com.duckduckgo.pir.impl.models.ExtractedProfile import com.duckduckgo.pir.impl.models.ProfileQuery import com.duckduckgo.pir.impl.models.scheduling.JobRecord @@ -592,18 +591,15 @@ class RealJobRecordUpdater @Inject constructor( } } + /** + * Mirrors the unique index of `pir_extracted_profiles`, so two profiles compare equal exactly when + * the DB considers them the same row. Scraped fields are deliberately excluded: they get refreshed + * on re-scan, so a changed relative or address must not read as a removal. + */ private data class ExtractedProfileComparisonKey( val profileQueryId: Long, val brokerName: String, val name: String, - val alternativeNames: List, - val age: String, - val addresses: List, - val phoneNumbers: List, - val relatives: List, - val reportId: String, - val email: String, - val fullName: String, val profileUrl: String, val identifier: String, ) @@ -613,14 +609,6 @@ class RealJobRecordUpdater @Inject constructor( profileQueryId = profileQueryId, brokerName = brokerName, name = name, - alternativeNames = alternativeNames, - age = age, - addresses = addresses, - phoneNumbers = phoneNumbers, - relatives = relatives, - reportId = reportId, - email = email, - fullName = fullName, profileUrl = profileUrl, identifier = identifier, ) diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scripts/models/PirScriptRequestParams.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scripts/models/PirScriptRequestParams.kt index c595e6b59e1d..35fb1078b654 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scripts/models/PirScriptRequestParams.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scripts/models/PirScriptRequestParams.kt @@ -46,6 +46,21 @@ data class ExtractedProfileParams( val profileUrl: String? = null, val email: String? = null, val fullName: String? = null, + val age: String? = null, + val addresses: List? = null, + val phoneNumbers: List? = null, + val relatives: List? = null, + val alternativeNames: List? = null, + val identifier: String? = null, + val extras: Map? = null, +) + +// Nullable extras (unlike the domain model's non-null map) so Moshi omits the key instead of emitting an empty object +data class AddressCityStateParams( + val city: String, + val state: String, + val fullAddress: String? = null, + val extras: Map? = null, ) // Wraps the generated email as { "email": "..." } so C-S-S can resolve it via data[dataSource].email. diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scripts/models/PirScriptResponseParams.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scripts/models/PirScriptResponseParams.kt index b9b2ec50c46a..b5e72ef5a9df 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scripts/models/PirScriptResponseParams.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scripts/models/PirScriptResponseParams.kt @@ -69,12 +69,16 @@ sealed class PirSuccessResponse( val email: String? = null, val removedDate: String? = null, val fullName: String? = null, + // Open bag; typed as Any? because a stricter value type would make one unexpected value + // from a broker config abort the whole extract response. Narrowed to strings on the way in. + val extras: Map? = null, ) data class ScriptAddressCityState( val city: String, val state: String, val fullAddress: String? = null, + val extras: Map? = null, ) } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDatabase.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDatabase.kt index be7a36a4026b..d90cf0507598 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDatabase.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDatabase.kt @@ -54,7 +54,7 @@ import com.squareup.moshi.Types @Database( exportSchema = true, - version = 16, + version = 17, entities = [ BrokerJsonEtag::class, BrokerEntity::class, @@ -77,6 +77,8 @@ import com.squareup.moshi.Types autoMigrations = [ // v16: adds nullable optOutFormSubmittedDate column to OptOutJobRecordEntity AutoMigration(from = 15, to = 16), + // v17: adds extras column (defaulting to "{}") to StoredExtractedProfile + AutoMigration(from = 16, to = 17), ], ) @TypeConverters(PirDatabaseConverters::class) @@ -108,6 +110,9 @@ object PirDatabaseConverters { private val stringListType = Types.newParameterizedType(List::class.java, String::class.java) private val stringListAdapter: JsonAdapter> = Moshi.Builder().build().adapter(stringListType) + private val stringMapType = Types.newParameterizedType(Map::class.java, String::class.java, String::class.java) + private val stringMapAdapter: JsonAdapter> = Moshi.Builder().build().adapter(stringMapType) + @TypeConverter @JvmStatic fun toStringList(value: String): List = stringListAdapter.fromJson(value)!! @@ -115,4 +120,12 @@ object PirDatabaseConverters { @TypeConverter @JvmStatic fun fromStringList(value: List): String = stringListAdapter.toJson(value) + + @TypeConverter + @JvmStatic + fun toStringMap(value: String): Map = stringMapAdapter.fromJson(value) ?: emptyMap() + + @TypeConverter + @JvmStatic + fun fromStringMap(value: Map): String = stringMapAdapter.toJson(value) } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirEventsRepository.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirEventsRepository.kt index 337c36dff71d..d60611dfb978 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirEventsRepository.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirEventsRepository.kt @@ -50,6 +50,7 @@ import kotlinx.coroutines.withContext import logcat.LogPriority.ERROR import logcat.logcat import javax.inject.Inject +import javax.inject.Named interface PirEventsRepository { fun getAllEventLogsFlow(): Flow> @@ -121,7 +122,8 @@ interface PirEventsRepository { ) @SingleInstanceIn(AppScope::class) class RealPirEventsRepository @Inject constructor( - val moshi: Moshi, + // Kotlin-aware, so reading a log back respects the non-null defaults on ExtractedProfile + @Named("pir") val moshi: Moshi, private val dispatcherProvider: DispatcherProvider, private val databaseFactory: PirSecureStorageDatabaseFactory, @AppCoroutineScope private val appCoroutineScope: CoroutineScope, diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirRepository.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirRepository.kt index ea8ef5866978..82c00e48a9fa 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirRepository.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirRepository.kt @@ -48,6 +48,7 @@ import com.duckduckgo.pir.impl.store.db.UserProfile import com.duckduckgo.pir.impl.store.db.UserProfileDao import com.duckduckgo.pir.impl.store.secure.PirSecureStorageDatabaseFactory import com.squareup.moshi.Moshi +import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.CoroutineStart import kotlinx.coroutines.Deferred @@ -138,7 +139,10 @@ interface PirRepository { /** * This method saves the new extracted profiles to the database. - * Any existing profiles (see indices of the table), we ignore them + * + * Profiles whose identity (see indices of the table) is already stored are refreshed in place: the + * scraped fields are overwritten with the newly extracted values while the row id, the date it was + * first found and the deprecated flag are preserved. Nothing is stored for a deprecated profile query. */ suspend fun saveNewExtractedProfiles(extractedProfiles: List) @@ -318,7 +322,10 @@ class RealPirRepository( prepareDatabase() } - private val addressCityStateAdapter by lazy { Moshi.Builder().build().adapter(AddressCityState::class.java) } + // Kotlin-aware so legacy stored JSON without the extras key falls back to the declared default instead of null + private val addressCityStateAdapter by lazy { + Moshi.Builder().add(KotlinJsonAdapterFactory()).build().adapter(AddressCityState::class.java) + } override suspend fun isRepositoryAvailable(): Boolean = database.await() != null @@ -556,12 +563,30 @@ class RealPirRepository( return@withContext } - extractedProfiles - .map { - it.toStoredExtractedProfile() - }.also { - extractedProfileDao()?.insertNewExtractedProfiles(it) + val dao = extractedProfileDao() ?: return@withContext + + val storedByIdentity = + extractedProfiles + .map { it.brokerName to it.profileQueryId } + .distinct() + .flatMap { (brokerName, queryId) -> + dao.getExtractedProfilesForBrokerAndProfile(brokerName, queryId) + }.associateBy { it.identity() } + + val toInsert = mutableListOf() + val toRefresh = mutableListOf() + + extractedProfiles.forEach { extractedProfile -> + val incoming = extractedProfile.toStoredExtractedProfile() + val stored = storedByIdentity[incoming.identity()] + if (stored == null) { + toInsert.add(incoming) + } else { + toRefresh.add(stored.withScrapedFieldsFrom(incoming)) } + } + + dao.insertAndRefreshExtractedProfiles(toInsert = toInsert, toRefresh = toRefresh) } } @@ -869,6 +894,40 @@ class RealPirRepository( }, ) + /** + * The tuple covered by the unique index of `pir_extracted_profiles`: a change to any of these fields + * describes a different profile, so it is stored as a new row rather than refreshing an existing one. + */ + private data class ExtractedProfileIdentity( + val profileQueryId: Long, + val brokerName: String, + val name: String, + val profileUrl: String, + val identifier: String, + ) + + private fun StoredExtractedProfile.identity(): ExtractedProfileIdentity = + ExtractedProfileIdentity( + profileQueryId = profileQueryId, + brokerName = brokerName, + name = name, + profileUrl = profileUrl, + identifier = identifier, + ) + + private fun StoredExtractedProfile.withScrapedFieldsFrom(incoming: StoredExtractedProfile): StoredExtractedProfile = + copy( + alternativeNames = incoming.alternativeNames, + age = incoming.age, + addresses = incoming.addresses, + phoneNumbers = incoming.phoneNumbers, + relatives = incoming.relatives, + reportId = incoming.reportId, + email = incoming.email, + fullName = incoming.fullName, + extras = incoming.extras, + ) + private fun StoredExtractedProfile.toExtractedProfile(): ExtractedProfile = ExtractedProfile( dbId = this.id, @@ -890,6 +949,7 @@ class RealPirRepository( fullName = this.fullName, dateAddedInMillis = this.dateAddedInMillis, deprecated = this.deprecated, + extras = this.extras, ) private fun ExtractedProfile.toStoredExtractedProfile(): StoredExtractedProfile = @@ -918,6 +978,7 @@ class RealPirRepository( this.dateAddedInMillis }, deprecated = this.deprecated, + extras = this.extras, ) private fun ProfileQuery.toUserProfile(): UserProfile = diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/ExtractedProfileDao.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/ExtractedProfileDao.kt index 1f2b89d82eb2..313ee9fe5135 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/ExtractedProfileDao.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/ExtractedProfileDao.kt @@ -20,6 +20,8 @@ import androidx.room.Dao import androidx.room.Insert import androidx.room.OnConflictStrategy import androidx.room.Query +import androidx.room.Transaction +import androidx.room.Update import kotlinx.coroutines.flow.Flow @Dao @@ -51,6 +53,22 @@ interface ExtractedProfileDao { @Insert(onConflict = OnConflictStrategy.IGNORE) fun insertNewExtractedProfiles(extractedProfiles: List) + /** + * Matches on the [StoredExtractedProfile.id] primary key, so the row - and every job record keyed on + * it - survives. A REPLACE-based insert would instead delete and re-insert, orphaning those records. + */ + @Update + fun updateExtractedProfiles(extractedProfiles: List) + + @Transaction + fun insertAndRefreshExtractedProfiles( + toInsert: List, + toRefresh: List, + ) { + insertNewExtractedProfiles(toInsert) + updateExtractedProfiles(toRefresh) + } + @Query("DELETE from pir_extracted_profiles") fun deleteAllExtractedProfiles() diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/ExtractedProfileEntities.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/ExtractedProfileEntities.kt index 9a5f934beeb2..ebcb6247668e 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/ExtractedProfileEntities.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/ExtractedProfileEntities.kt @@ -16,6 +16,7 @@ package com.duckduckgo.pir.impl.store.db +import androidx.room.ColumnInfo import androidx.room.Entity import androidx.room.Index import androidx.room.PrimaryKey @@ -52,4 +53,7 @@ data class StoredExtractedProfile( val fullName: String = "", val dateAddedInMillis: Long = 0L, // Tells us when the extracted profile has been found val deprecated: Boolean = false, // This should tell us if the profile is irrelevant for PIR (this is not me, profile edits) + // Open bag of profile fields C-S-S extracted that native doesn't model, stored verbatim and forwarded on opt-out + @ColumnInfo(defaultValue = "{}") + val extras: Map = emptyMap(), ) diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/common/PirUtilsTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/common/PirUtilsTest.kt index 0e6ed93c6f8a..eebf693963be 100644 --- a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/common/PirUtilsTest.kt +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/common/PirUtilsTest.kt @@ -18,6 +18,7 @@ package com.duckduckgo.pir.impl.common import com.duckduckgo.pir.impl.models.AddressCityState import com.duckduckgo.pir.impl.models.ExtractedProfile +import com.duckduckgo.pir.impl.scripts.models.AddressCityStateParams import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse import org.junit.Assert.assertNull @@ -129,6 +130,71 @@ class PirUtilsTest { assertNull(result.profileUrl) assertNull(result.fullName) assertNull(result.email) + assertNull(result.age) + assertNull(result.addresses) + assertNull(result.phoneNumbers) + assertNull(result.relatives) + assertNull(result.alternativeNames) + assertNull(result.identifier) + assertNull(result.extras) + } + + @Test + fun whenToParamsWithFullRecordThenPassesEveryFieldThrough() { + val profile = ExtractedProfile( + profileQueryId = 123L, + brokerName = "test-broker", + name = "John Doe", + alternativeNames = listOf("Johnny"), + age = "35", + addresses = listOf( + AddressCityState( + city = "City", + state = "State", + fullAddress = "123 Main St", + extras = mapOf("county" to "Cook"), + ), + ), + phoneNumbers = listOf("555-1234"), + relatives = listOf("Jane Doe"), + identifier = "id-123", + extras = mapOf("middleInitial" to "M"), + ) + + val result = profile.toParams("John Doe") + + assertEquals("35", result.age) + assertEquals(listOf("555-1234"), result.phoneNumbers) + assertEquals(listOf("Jane Doe"), result.relatives) + assertEquals(listOf("Johnny"), result.alternativeNames) + assertEquals("id-123", result.identifier) + assertEquals(mapOf("middleInitial" to "M"), result.extras) + assertEquals( + listOf( + AddressCityStateParams( + city = "City", + state = "State", + fullAddress = "123 Main St", + extras = mapOf("county" to "Cook"), + ), + ), + result.addresses, + ) + } + + @Test + fun whenToParamsWithEmptyExtrasThenExtrasAreNullAtBothLevels() { + val profile = ExtractedProfile( + profileQueryId = 123L, + brokerName = "test-broker", + name = "John Doe", + addresses = listOf(AddressCityState(city = "City", state = "State")), + ) + + val result = profile.toParams("John Doe") + + assertNull(result.extras) + assertNull(result.addresses!!.single().extras) } @Test @@ -473,6 +539,57 @@ class PirUtilsTest { assertTrue(result) } + @Test + fun whenMatchesWithAddressesDifferingOnlyByExtrasThenReturnsTrue() { + val profile1 = ExtractedProfile( + profileQueryId = 123L, + brokerName = "test-broker", + name = "John Doe", + age = "35", + addresses = listOf(AddressCityState(city = "City", state = "State", fullAddress = "123 Main St")), + ) + + val profile2 = ExtractedProfile( + profileQueryId = 456L, + brokerName = "test-broker", + name = "John Doe", + age = "35", + addresses = listOf( + AddressCityState( + city = "City", + state = "State", + fullAddress = "123 Main St", + extras = mapOf("county" to "Cook"), + ), + ), + ) + + assertTrue(profile1.matches(profile2)) + } + + @Test + fun whenMatchesWithDifferentAddressesThenReturnsFalseEvenWithMatchingExtras() { + val extras = mapOf("county" to "Cook") + + val profile1 = ExtractedProfile( + profileQueryId = 123L, + brokerName = "test-broker", + name = "John Doe", + age = "35", + addresses = listOf(AddressCityState(city = "City", state = "State", fullAddress = "123 Main St", extras = extras)), + ) + + val profile2 = ExtractedProfile( + profileQueryId = 456L, + brokerName = "test-broker", + name = "John Doe", + age = "35", + addresses = listOf(AddressCityState(city = "Other City", state = "State", fullAddress = "456 Oak Ave", extras = extras)), + ) + + assertFalse(profile1.matches(profile2)) + } + @Test fun whenMatchesWithComplexProfilesThenEvaluatesCorrectly() { val address1 = AddressCityState(city = "City", state = "State", fullAddress = "123 Main St") diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/common/RealPirRunStateHandlerTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/common/RealPirRunStateHandlerTest.kt index cac80880357e..42306c25e644 100644 --- a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/common/RealPirRunStateHandlerTest.kt +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/common/RealPirRunStateHandlerTest.kt @@ -59,10 +59,12 @@ import com.duckduckgo.pir.impl.store.db.EmailConfirmationEventType.EMAIL_CONFIRM import com.duckduckgo.pir.impl.store.db.PirBrokerScanLog import com.squareup.moshi.Moshi import kotlinx.coroutines.test.runTest +import org.junit.Assert.assertEquals import org.junit.Before import org.junit.Rule import org.junit.Test import org.mockito.kotlin.any +import org.mockito.kotlin.argumentCaptor import org.mockito.kotlin.inOrder import org.mockito.kotlin.mock import org.mockito.kotlin.verify @@ -145,6 +147,7 @@ class RealPirRunStateHandlerTest { city = "New York", state = "NY", fullAddress = "123 Main St", + extras = mapOf("county" to "Kings"), ), ), phoneNumbers = listOf("555-1234"), @@ -154,6 +157,7 @@ class RealPirRunStateHandlerTest { reportId = "report123", email = "john@example.com", fullName = "John Michael Doe", + extras = mapOf("middleInitial" to "M"), ) private val testEmailConfirmationJob = @@ -371,6 +375,7 @@ class RealPirRunStateHandlerTest { city = "New York", state = "NY", fullAddress = "123 Main St", + extras = mapOf("county" to "Kings"), ), ), phoneNumbers = listOf("555-1234"), @@ -379,6 +384,7 @@ class RealPirRunStateHandlerTest { reportId = "report123", email = "john@example.com", fullName = "John Michael Doe", + extras = mapOf("middleInitial" to "M"), ) whenever(mockJobRecordUpdater.markReappearedOptOutJobRecords(any(), any(), any())) .thenReturn(emptyList()) @@ -433,6 +439,7 @@ class RealPirRunStateHandlerTest { city = "New York", state = "NY", fullAddress = "123 Main St", + extras = mapOf("county" to "Kings"), ), ), phoneNumbers = listOf("555-1234"), @@ -441,6 +448,7 @@ class RealPirRunStateHandlerTest { reportId = "report123", email = "john@example.com", fullName = "John Michael Doe", + extras = mapOf("middleInitial" to "M"), ) val revertedRecord = OptOutJobRecord( @@ -467,6 +475,54 @@ class RealPirRunStateHandlerTest { verify(mockPixelSender).reportBrokerOptOutProfileReappeared(testBroker.url) } + @Test + fun whenHandleBrokerScanActionSucceededWithUnexpectedlyShapedExtrasThenKeepsOnlyStringValues() = + runTest { + // extras is string values only by contract; anything else is dropped rather than coerced, + // so a broker config emitting an unexpected value can never poison the stored record. + val extractedResponse = + ExtractedResponse( + actionID = "extract123", + actionType = "extract", + response = listOf( + testScriptExtractedProfile.copy( + extras = mapOf( + "county" to "Cook", + "verified" to true, + "count" to 3, + "missing" to null, + "nested" to mapOf("a" to "b"), + "list" to listOf("a"), + ), + addresses = listOf( + ScriptAddressCityState( + city = "New York", + state = "NY", + fullAddress = "123 Main St", + extras = null, + ), + ), + ), + ), + ) + val state = + BrokerScanActionSucceeded( + broker = testBroker, + profileQueryId = testProfileQueryId, + pirSuccessResponse = extractedResponse, + ) + whenever(mockJobRecordUpdater.markReappearedOptOutJobRecords(any(), any(), any())) + .thenReturn(emptyList()) + + testee.handleState(state) + + val captor = argumentCaptor>() + verify(mockRepository).saveNewExtractedProfiles(captor.capture()) + val saved = captor.firstValue.single() + assertEquals(mapOf("county" to "Cook"), saved.extras) + assertEquals(emptyMap(), saved.addresses.single().extras) + } + @Test fun whenHandleBrokerScanActionSucceededWithExtractedResponseAndNoProfilesFoundThenUpdatesJobAsNoMatch() = runTest { diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/common/actions/ExecuteBrokerStepActionEventHandlerTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/common/actions/ExecuteBrokerStepActionEventHandlerTest.kt index c635ca526857..1fe98039e3a3 100644 --- a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/common/actions/ExecuteBrokerStepActionEventHandlerTest.kt +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/common/actions/ExecuteBrokerStepActionEventHandlerTest.kt @@ -38,6 +38,7 @@ import com.duckduckgo.pir.impl.common.actions.PirActionsRunnerStateEngine.SideEf import com.duckduckgo.pir.impl.common.actions.PirActionsRunnerStateEngine.SideEffect.LoadUrl import com.duckduckgo.pir.impl.common.actions.PirActionsRunnerStateEngine.SideEffect.PushJsAction import com.duckduckgo.pir.impl.common.actions.PirActionsRunnerStateEngine.State +import com.duckduckgo.pir.impl.models.AddressCityState import com.duckduckgo.pir.impl.models.Broker import com.duckduckgo.pir.impl.models.ExtractedProfile import com.duckduckgo.pir.impl.models.ProfileQuery @@ -46,6 +47,7 @@ import com.duckduckgo.pir.impl.models.scheduling.JobRecord.EmailConfirmationJobR import com.duckduckgo.pir.impl.models.scheduling.JobRecord.EmailConfirmationJobRecord.JobAttemptData import com.duckduckgo.pir.impl.models.scheduling.JobRecord.EmailConfirmationJobRecord.LinkFetchData import com.duckduckgo.pir.impl.pixels.PirStage +import com.duckduckgo.pir.impl.scripts.models.AddressCityStateParams import com.duckduckgo.pir.impl.scripts.models.BrokerAction import com.duckduckgo.pir.impl.scripts.models.DataSource import com.duckduckgo.pir.impl.scripts.models.ElementSelector @@ -109,6 +111,10 @@ class ExecuteBrokerStepActionEventHandlerTest { brokerName = testBrokerName, name = "John Doe", email = "john@example.com", + addresses = listOf( + AddressCityState(city = "New York", state = "NY", extras = mapOf("county" to "Kings")), + ), + extras = mapOf("middleInitial" to "M"), ) private val testEmailConfirmationJob = @@ -689,6 +695,11 @@ class ExecuteBrokerStepActionEventHandlerTest { assertEquals(testProfileQuery, userData.userProfile) assertEquals("John Doe", userData.extractedProfile?.name) assertEquals("john@example.com", userData.extractedProfile?.email) + assertEquals(mapOf("middleInitial" to "M"), userData.extractedProfile?.extras) + assertEquals( + listOf(AddressCityStateParams(city = "New York", state = "NY", extras = mapOf("county" to "Kings"))), + userData.extractedProfile?.addresses, + ) } @Test diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/scheduling/RealJobRecordUpdaterTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/scheduling/RealJobRecordUpdaterTest.kt index b46b1b79f0a8..0b2041577667 100644 --- a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/scheduling/RealJobRecordUpdaterTest.kt +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/scheduling/RealJobRecordUpdaterTest.kt @@ -18,6 +18,7 @@ package com.duckduckgo.pir.impl.scheduling import com.duckduckgo.common.test.CoroutineTestRule import com.duckduckgo.common.utils.CurrentTimeProvider +import com.duckduckgo.pir.impl.models.AddressCityState import com.duckduckgo.pir.impl.models.ExtractedProfile import com.duckduckgo.pir.impl.models.scheduling.JobRecord.EmailConfirmationJobRecord import com.duckduckgo.pir.impl.models.scheduling.JobRecord.EmailConfirmationJobRecord.EmailData @@ -516,6 +517,115 @@ class RealJobRecordUpdaterTest { verify(mockSchedulingRepository, never()).saveOptOutJobRecord(any()) } + @Test + fun whenMarkRemovedProfilesWithChangedScrapedFieldsButSameIdentityThenDoesNotMarkAsRemoved() = + runTest { + // A re-scan of a still-listed profile that picked up a new relative, address and extras. + // Identity is unchanged, so this is a refresh, not a removal. + val storedProfiles = listOf(testExtractedProfile1) + val newProfiles = listOf( + testExtractedProfile1.copy( + dbId = 0L, + relatives = listOf("Jane Doe"), + addresses = listOf(AddressCityState(city = "City", state = "State")), + alternativeNames = listOf("Johnny"), + age = "36", + extras = mapOf("county" to "Cook"), + ), + ) + + whenever(mockRepository.getExtractedProfiles(testBrokerName, testProfileQueryId)) + .thenReturn(storedProfiles) + + toTest.markRemovedOptOutJobRecords(newProfiles, testBrokerName, testProfileQueryId) + + verify(mockSchedulingRepository, never()).getValidOptOutJobRecord(any(), any()) + verify(mockSchedulingRepository, never()).saveOptOutJobRecord(any()) + } + + @Test + fun whenMarkReappearedProfilesWithChangedScrapedFieldsButSameIdentityThenStillRevertsToRequested() = + runTest { + val storedProfiles = listOf(testExtractedProfile1) + val newProfiles = listOf( + testExtractedProfile1.copy( + dbId = 0L, + relatives = listOf("Jane Doe"), + extras = mapOf("county" to "Cook"), + ), + ) + + val removedOptOutJobRecord = testOptOutJobRecord.copy( + extractedProfileId = 100L, + status = OptOutJobStatus.REMOVED, + optOutRemovedDateInMillis = 3000L, + ) + + whenever(mockRepository.getExtractedProfiles(testBrokerName, testProfileQueryId)) + .thenReturn(storedProfiles) + whenever(mockSchedulingRepository.getValidOptOutJobRecord(100L)) + .thenReturn(removedOptOutJobRecord) + + val result = toTest.markReappearedOptOutJobRecords(newProfiles, testBrokerName, testProfileQueryId) + + val expectedRecord = removedOptOutJobRecord.copy( + status = OptOutJobStatus.REQUESTED, + optOutRemovedDateInMillis = 0L, + ) + verify(mockSchedulingRepository).saveOptOutJobRecord(expectedRecord) + assertEquals(listOf(expectedRecord), result) + } + + @Test + fun whenMarkRemovedProfilesWithChangedIdentifierThenMarksAsRemoved() = + runTest { + val storedProfiles = listOf(testExtractedProfile1) + val newProfiles = listOf(testExtractedProfile1.copy(dbId = 0L, identifier = "id999")) + val optOutJobRecord1 = testOptOutJobRecord.copy(extractedProfileId = 100L) + + whenever(mockRepository.getUserProfileQuery(testProfileQueryId)).thenReturn(mock()) + whenever(mockRepository.getExtractedProfiles(testBrokerName, testProfileQueryId)) + .thenReturn(storedProfiles) + whenever(mockSchedulingRepository.getValidOptOutJobRecord(100L)) + .thenReturn(optOutJobRecord1) + + toTest.markRemovedOptOutJobRecords(newProfiles, testBrokerName, testProfileQueryId) + + verify(mockSchedulingRepository).saveOptOutJobRecord( + optOutJobRecord1.copy( + status = OptOutJobStatus.REMOVED, + optOutRemovedDateInMillis = TEST_CURRENT_TIME, + deprecated = false, + ), + ) + } + + @Test + fun whenUpdateScanMatchesFoundWithDeprecatedProfileAndChangedScrapedFieldsThenDoesNotMarkAsDeprecated() = + runTest { + val deprecatedProfile = mock() + whenever(deprecatedProfile.deprecated).thenReturn(true) + whenever(mockRepository.getUserProfileQuery(testProfileQueryId)).thenReturn(deprecatedProfile) + + whenever(mockRepository.getExtractedProfiles(testBrokerName, testProfileQueryId)) + .thenReturn(listOf(testExtractedProfile1)) + + // Same profile still listed, only its scraped fields changed - not a confirmed removal + val newProfiles = listOf( + testExtractedProfile1.copy(dbId = 0L, extras = mapOf("county" to "Cook")), + ) + + toTest.updateScanMatchesFound(newProfiles, testBrokerName, testProfileQueryId) + + verify(mockSchedulingRepository).updateScanJobRecordStatus( + newStatus = ScanJobStatus.MATCHES_FOUND, + newLastScanDateMillis = TEST_CURRENT_TIME, + brokerName = testBrokerName, + profileQueryId = testProfileQueryId, + deprecated = false, + ) + } + @Test fun whenMarkReappearedProfilesWithReappearedRemovedProfileThenRevertsToRequestedAndClearsRemovedDate() = runTest { diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/scripts/RealBrokerActionProcessorTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/scripts/RealBrokerActionProcessorTest.kt index a5d92f2a3364..458afbd6186d 100644 --- a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/scripts/RealBrokerActionProcessorTest.kt +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/scripts/RealBrokerActionProcessorTest.kt @@ -21,7 +21,9 @@ import androidx.test.ext.junit.runners.AndroidJUnit4 import com.duckduckgo.js.messaging.api.JsMessageCallback import com.duckduckgo.js.messaging.api.JsMessaging import com.duckduckgo.js.messaging.api.SubscriptionEventData +import com.duckduckgo.pir.impl.scripts.models.AddressCityStateParams import com.duckduckgo.pir.impl.scripts.models.BrokerAction +import com.duckduckgo.pir.impl.scripts.models.ExtractedProfileParams import com.duckduckgo.pir.impl.scripts.models.PirError import com.duckduckgo.pir.impl.scripts.models.PirError.ActionError.JsActionFailed import com.duckduckgo.pir.impl.scripts.models.PirScriptRequestData @@ -42,7 +44,9 @@ import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory import kotlinx.coroutines.test.runTest import org.json.JSONObject import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull import org.junit.Before import org.junit.Test import org.junit.runner.RunWith @@ -176,6 +180,189 @@ class RealBrokerActionProcessorTest { assertEquals("navigate", successCaptor.firstValue.actionType) } + @Test + fun whenProcessExtractResponseWithExtrasThenParsesThemAtBothLevels() = runTest { + val callback = registerAndGetCallback() + + val successJson = """ + { + "result": { + "success": { + "actionID": "action-1", + "actionType": "extract", + "response": [ + { + "name": "John Doe", + "profileUrl": "https://example.com/profile", + "identifier": "id-123", + "addresses": [ + { + "city": "New York", + "state": "NY", + "extras": { "county": "Kings" } + } + ], + "extras": { "middleInitial": "M" } + } + ] + } + } + } + """.trimIndent() + + callback.process( + featureName = PIRScriptConstants.SCRIPT_FEATURE_NAME, + method = PIRScriptConstants.RECEIVED_METHOD_NAME_COMPLETED, + id = null, + data = JSONObject(successJson), + ) + + val successCaptor = argumentCaptor() + verify(mockActionResultListener).onSuccess(successCaptor.capture()) + val profile = (successCaptor.firstValue as ExtractedResponse).response.single() + assertEquals(mapOf("middleInitial" to "M"), profile.extras) + assertEquals(mapOf("county" to "Kings"), profile.addresses.single().extras) + } + + @Test + fun whenProcessExtractResponseWithoutExtrasKeysThenExtrasAreEmpty() = runTest { + val callback = registerAndGetCallback() + + // Payload from a C-S-S version that predates extras - must parse, not fail + val successJson = """ + { + "result": { + "success": { + "actionID": "action-1", + "actionType": "extract", + "response": [ + { + "name": "John Doe", + "profileUrl": "https://example.com/profile", + "identifier": "id-123", + "addresses": [ + { "city": "New York", "state": "NY" } + ] + } + ] + } + } + } + """.trimIndent() + + callback.process( + featureName = PIRScriptConstants.SCRIPT_FEATURE_NAME, + method = PIRScriptConstants.RECEIVED_METHOD_NAME_COMPLETED, + id = null, + data = JSONObject(successJson), + ) + + val successCaptor = argumentCaptor() + verify(mockActionResultListener).onSuccess(successCaptor.capture()) + val profile = (successCaptor.firstValue as ExtractedResponse).response.single() + assertNull(profile.extras) + assertNull(profile.addresses.single().extras) + } + + @Test + fun whenProcessExtractResponseWithUnexpectedlyShapedExtrasThenResponseStillParses() = runTest { + val callback = registerAndGetCallback() + + // extras is documented as string values only. A broker config emitting anything else is + // undefined behaviour, but it must not cost us the whole response. + val successJson = """ + { + "result": { + "success": { + "actionID": "action-1", + "actionType": "extract", + "response": [ + { + "name": "John Doe", + "profileUrl": "https://example.com/profile", + "identifier": "id-123", + "addresses": [ + { "city": "New York", "state": "NY", "extras": null } + ], + "extras": { + "county": "Cook", + "verified": true, + "count": 3, + "missing": null, + "nested": { "a": "b" }, + "list": ["a"] + } + } + ] + } + } + } + """.trimIndent() + + callback.process( + featureName = PIRScriptConstants.SCRIPT_FEATURE_NAME, + method = PIRScriptConstants.RECEIVED_METHOD_NAME_COMPLETED, + id = null, + data = JSONObject(successJson), + ) + + val successCaptor = argumentCaptor() + verify(mockActionResultListener).onSuccess(successCaptor.capture()) + val profile = (successCaptor.firstValue as ExtractedResponse).response.single() + assertEquals("John Doe", profile.name) + assertEquals("Cook", profile.extras?.get("county")) + assertNull(profile.addresses.single().extras) + } + + @Test + fun whenPushActionWithExtrasThenExtrasAreSerializedAtBothLevels() = runTest { + testee.pushAction( + BrokerAction.FillForm(id = "action-fill", elements = emptyList(), selector = "form"), + UserProfile( + extractedProfile = ExtractedProfileParams( + name = "John Doe", + addresses = listOf( + AddressCityStateParams(city = "New York", state = "NY", extras = mapOf("county" to "Kings")), + ), + extras = mapOf("middleInitial" to "M"), + ), + ), + ) + + val extractedProfileJson = capturedExtractedProfileJson() + assertEquals("M", extractedProfileJson.getJSONObject("extras").getString("middleInitial")) + assertEquals( + "Kings", + extractedProfileJson.getJSONArray("addresses").getJSONObject(0).getJSONObject("extras").getString("county"), + ) + } + + @Test + fun whenPushActionWithEmptyExtrasThenNoExtrasKeyIsSerialized() = runTest { + testee.pushAction( + BrokerAction.FillForm(id = "action-fill", elements = emptyList(), selector = "form"), + UserProfile( + extractedProfile = ExtractedProfileParams( + name = "John Doe", + addresses = listOf(AddressCityStateParams(city = "New York", state = "NY")), + ), + ), + ) + + val extractedProfileJson = capturedExtractedProfileJson() + assertFalse(extractedProfileJson.has("extras")) + assertFalse(extractedProfileJson.getJSONArray("addresses").getJSONObject(0).has("extras")) + } + + private fun capturedExtractedProfileJson(): JSONObject { + val eventCaptor = argumentCaptor() + verify(mockJsMessaging).sendSubscriptionEvent(eventCaptor.capture()) + return eventCaptor.firstValue.params + .getJSONObject("state") + .getJSONObject("data") + .getJSONObject("extractedProfile") + } + @Test fun whenProcessJsCallbackWithErrorThenCallsOnError() = runTest { val callback = registerAndGetCallback() diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/PirDatabaseMigration16To17Test.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/PirDatabaseMigration16To17Test.kt new file mode 100644 index 000000000000..0300e7701b7c --- /dev/null +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/PirDatabaseMigration16To17Test.kt @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2026 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.pir.impl.store + +import androidx.room.testing.MigrationTestHelper +import androidx.sqlite.db.SupportSQLiteDatabase +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import org.junit.Assert.assertEquals +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +class PirDatabaseMigration16To17Test { + @get:Rule + val testHelper = MigrationTestHelper( + InstrumentationRegistry.getInstrumentation(), + PirDatabase::class.java, + emptyList(), + ) + + @Test + fun `migration preserves existing extracted profiles and defaults extras to an empty map`() { + testHelper.createDatabase(TEST_DB_NAME, 16).use { db -> + db.insertExtractedProfile( + id = 1, + // address JSON as written before extras existed + addresses = """["{\"city\":\"New York\",\"state\":\"NY\"}"]""", + ) + } + + testHelper.runMigrationsAndValidate(TEST_DB_NAME, 17, true).use { db -> + assertEquals(1L, db.rowCount()) + assertEquals("""["{\"city\":\"New York\",\"state\":\"NY\"}"]""", db.readColumn(id = 1, column = "addresses")) + assertEquals("John Doe", db.readColumn(id = 1, column = "name")) + assertEquals("identifier-123", db.readColumn(id = 1, column = "identifier")) + assertEquals("1000", db.readColumn(id = 1, column = "dateAddedInMillis")) + assertEquals("{}", db.readColumn(id = 1, column = "extras")) + } + } + + private companion object { + const val TEST_DB_NAME = "pir_migration_test" + } +} + +private fun SupportSQLiteDatabase.insertExtractedProfile( + id: Long, + addresses: String, +) { + execSQL( + """ + INSERT INTO pir_extracted_profiles ( + id, profileQueryId, brokerName, name, alternativeNames, age, addresses, phoneNumbers, + relatives, profileUrl, identifier, reportId, email, fullName, dateAddedInMillis, deprecated + ) VALUES ( + ?, 1, 'TestBroker', 'John Doe', '[]', '35', ?, '[]', + '[]', 'https://example.com/profile', 'identifier-123', 'report-1', 'john@example.com', + 'John Doe', 1000, 0 + ) + """.trimIndent(), + arrayOf(id, addresses), + ) +} + +private fun SupportSQLiteDatabase.readColumn( + id: Long, + column: String, +): String? = + query("SELECT $column FROM pir_extracted_profiles WHERE id = ?", arrayOf(id)).use { cursor -> + cursor.moveToFirst() + if (cursor.isNull(0)) null else cursor.getString(0) + } + +private fun SupportSQLiteDatabase.rowCount(): Long = + query("SELECT COUNT(*) FROM pir_extracted_profiles").use { cursor -> + cursor.moveToFirst() + cursor.getLong(0) + } diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/RealPirRepositoryTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/RealPirRepositoryTest.kt index 1898b59af45f..9c0d4086dd67 100644 --- a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/RealPirRepositoryTest.kt +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/RealPirRepositoryTest.kt @@ -29,6 +29,7 @@ import com.duckduckgo.pir.impl.store.PirRepository.EmailConfirmationLinkFetchSta import com.duckduckgo.pir.impl.store.db.BrokerDao import com.duckduckgo.pir.impl.store.db.BrokerJsonDao import com.duckduckgo.pir.impl.store.db.ExtractedProfileDao +import com.duckduckgo.pir.impl.store.db.StoredExtractedProfile import com.duckduckgo.pir.impl.store.db.UserName import com.duckduckgo.pir.impl.store.db.UserProfile import com.duckduckgo.pir.impl.store.db.UserProfileDao @@ -42,6 +43,7 @@ import org.junit.Before import org.junit.Rule import org.junit.Test import org.mockito.kotlin.any +import org.mockito.kotlin.argumentCaptor import org.mockito.kotlin.mock import org.mockito.kotlin.never import org.mockito.kotlin.verify @@ -673,7 +675,7 @@ class RealPirRepositoryTest { // Then verify(mockUserProfileDao).getUserProfile(profileQueryId) - verify(mockExtractedProfileDao).insertNewExtractedProfiles(any()) + verify(mockExtractedProfileDao).insertAndRefreshExtractedProfiles(any(), any()) } @Test @@ -702,7 +704,7 @@ class RealPirRepositoryTest { // Then verify(mockUserProfileDao).getUserProfile(profileQueryId) - verify(mockExtractedProfileDao, never()).insertNewExtractedProfiles(any()) + verify(mockExtractedProfileDao, never()).insertAndRefreshExtractedProfiles(any(), any()) } @Test @@ -740,7 +742,7 @@ class RealPirRepositoryTest { // Then verify(mockUserProfileDao).getUserProfile(profileQueryId) - verify(mockExtractedProfileDao).insertNewExtractedProfiles(any()) + verify(mockExtractedProfileDao).insertAndRefreshExtractedProfiles(any(), any()) } @Test @@ -772,7 +774,7 @@ class RealPirRepositoryTest { // Then verify(mockCurrentTimeProvider).currentTimeMillis() - verify(mockExtractedProfileDao).insertNewExtractedProfiles(any()) + verify(mockExtractedProfileDao).insertAndRefreshExtractedProfiles(any(), any()) } @Test @@ -804,7 +806,7 @@ class RealPirRepositoryTest { testee.saveNewExtractedProfiles(extractedProfiles) // Then - verify(mockExtractedProfileDao).insertNewExtractedProfiles(any()) + verify(mockExtractedProfileDao).insertAndRefreshExtractedProfiles(any(), any()) } @Test @@ -850,7 +852,216 @@ class RealPirRepositoryTest { // Then verify(mockUserProfileDao).getUserProfile(profileQueryId) - verify(mockExtractedProfileDao).insertNewExtractedProfiles(any()) + verify(mockExtractedProfileDao).insertAndRefreshExtractedProfiles(any(), any()) + } + + @Test + fun whenSaveNewExtractedProfilesWithExtrasThenExtrasArePersistedAtBothLevels() = runTest { + // Given + val profileQueryId = 1L + val extractedProfiles = listOf( + newExtractedProfile(profileQueryId).copy( + addresses = listOf( + AddressCityState(city = "New York", state = "NY", extras = mapOf("county" to "Kings")), + ), + extras = mapOf("middleInitial" to "M"), + ), + ) + whenever(mockUserProfileDao.getUserProfile(profileQueryId)).thenReturn(nonDeprecatedUserProfile(profileQueryId)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(1000000L) + + // When + testee.saveNewExtractedProfiles(extractedProfiles) + + // Then + val inserted = captureInserted().single() + assertEquals(mapOf("middleInitial" to "M"), inserted.extras) + assertTrue(inserted.addresses.single().contains("\"county\":\"Kings\"")) + } + + @Test + fun whenReadingExtractedProfileThenExtrasAreMappedBackAtBothLevels() = runTest { + // Given + whenever(mockExtractedProfileDao.getExtractedProfilesForBrokerAndProfile("TestBroker", 1L)) + .thenReturn( + listOf( + storedExtractedProfile( + addresses = listOf("""{"city":"New York","state":"NY","extras":{"county":"Kings"}}"""), + extras = mapOf("middleInitial" to "M"), + ), + ), + ) + + // When + val result = testee.getExtractedProfiles("TestBroker", 1L).single() + + // Then + assertEquals(mapOf("middleInitial" to "M"), result.extras) + assertEquals(mapOf("county" to "Kings"), result.addresses.single().extras) + } + + @Test + fun whenReadingLegacyAddressJsonWithoutExtrasThenExtrasIsEmpty() = runTest { + // Given - address JSON written before extras existed + whenever(mockExtractedProfileDao.getExtractedProfilesForBrokerAndProfile("TestBroker", 1L)) + .thenReturn( + listOf(storedExtractedProfile(addresses = listOf("""{"city":"New York","state":"NY"}"""))), + ) + + // When + val result = testee.getExtractedProfiles("TestBroker", 1L).single() + + // Then + assertEquals(emptyMap(), result.addresses.single().extras) + } + + @Test + fun whenSaveNewExtractedProfilesWithAlreadyStoredIdentityThenRefreshesRowInPlace() = runTest { + // Given + val profileQueryId = 1L + val stored = storedExtractedProfile( + id = 42L, + addresses = listOf("""{"city":"New York","state":"NY"}"""), + dateAddedInMillis = 111L, + ) + whenever(mockExtractedProfileDao.getExtractedProfilesForBrokerAndProfile("TestBroker", profileQueryId)) + .thenReturn(listOf(stored)) + whenever(mockUserProfileDao.getUserProfile(profileQueryId)).thenReturn(nonDeprecatedUserProfile(profileQueryId)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(999L) + + val rescanned = newExtractedProfile(profileQueryId).copy( + alternativeNames = listOf("Johnny"), + age = "36", + addresses = listOf(AddressCityState(city = "Boston", state = "MA")), + phoneNumbers = listOf("555-1234"), + relatives = listOf("Jane Doe"), + reportId = "report-999", + email = "new@example.com", + fullName = "John Michael Doe", + extras = mapOf("county" to "Cook"), + ) + + // When + testee.saveNewExtractedProfiles(listOf(rescanned)) + + // Then + assertTrue(captureInserted().isEmpty()) + val refreshed = captureRefreshed().single() + // identity and native-owned values are untouched + assertEquals(42L, refreshed.id) + assertEquals(111L, refreshed.dateAddedInMillis) + assertEquals(false, refreshed.deprecated) + // scraped fields are replaced + assertEquals(listOf("Johnny"), refreshed.alternativeNames) + assertEquals("36", refreshed.age) + assertEquals(listOf("555-1234"), refreshed.phoneNumbers) + assertEquals(listOf("Jane Doe"), refreshed.relatives) + assertEquals("report-999", refreshed.reportId) + assertEquals("new@example.com", refreshed.email) + assertEquals("John Michael Doe", refreshed.fullName) + assertEquals(mapOf("county" to "Cook"), refreshed.extras) + assertTrue(refreshed.addresses.single().contains("\"city\":\"Boston\"")) + } + + @Test + fun whenSaveNewExtractedProfilesWithNewIdentityThenInsertsInsteadOfRefreshing() = runTest { + // Given + val profileQueryId = 1L + whenever(mockExtractedProfileDao.getExtractedProfilesForBrokerAndProfile("TestBroker", profileQueryId)) + .thenReturn(listOf(storedExtractedProfile(id = 42L))) + whenever(mockUserProfileDao.getUserProfile(profileQueryId)).thenReturn(nonDeprecatedUserProfile(profileQueryId)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(999L) + + // When + testee.saveNewExtractedProfiles(listOf(newExtractedProfile(profileQueryId).copy(identifier = "identifier-999"))) + + // Then + assertTrue(captureRefreshed().isEmpty()) + val inserted = captureInserted().single() + assertEquals("identifier-999", inserted.identifier) + assertEquals(999L, inserted.dateAddedInMillis) + } + + @Test + fun whenSaveNewExtractedProfilesWithStoredDeprecatedRowThenRefreshesButKeepsDeprecated() = runTest { + // Given + val profileQueryId = 1L + whenever(mockExtractedProfileDao.getExtractedProfilesForBrokerAndProfile("TestBroker", profileQueryId)) + .thenReturn(listOf(storedExtractedProfile(id = 42L, deprecated = true))) + whenever(mockUserProfileDao.getUserProfile(profileQueryId)).thenReturn(nonDeprecatedUserProfile(profileQueryId)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(999L) + + // When + testee.saveNewExtractedProfiles( + listOf(newExtractedProfile(profileQueryId).copy(extras = mapOf("county" to "Cook"))), + ) + + // Then + val refreshed = captureRefreshed().single() + assertEquals(true, refreshed.deprecated) + assertEquals(mapOf("county" to "Cook"), refreshed.extras) + } + + @Test + fun whenSaveNewExtractedProfilesWithDeprecatedProfileQueryThenNeitherInsertsNorRefreshes() = runTest { + // Given + val profileQueryId = 1L + val deprecatedUserProfile = nonDeprecatedUserProfile(profileQueryId).copy(deprecated = true) + whenever(mockUserProfileDao.getUserProfile(profileQueryId)).thenReturn(deprecatedUserProfile) + + // When + testee.saveNewExtractedProfiles(listOf(newExtractedProfile(profileQueryId))) + + // Then + verify(mockExtractedProfileDao, never()).insertAndRefreshExtractedProfiles(any(), any()) + verify(mockExtractedProfileDao, never()).getExtractedProfilesForBrokerAndProfile(any(), any()) + } + + private fun newExtractedProfile(profileQueryId: Long) = ExtractedProfile( + dbId = 0L, + profileQueryId = profileQueryId, + brokerName = "TestBroker", + name = "John Michael Doe", + profileUrl = "https://example.com/profile", + identifier = "identifier-123", + ) + + private fun storedExtractedProfile( + id: Long = 1L, + addresses: List = emptyList(), + extras: Map = emptyMap(), + dateAddedInMillis: Long = 111L, + deprecated: Boolean = false, + ) = StoredExtractedProfile( + id = id, + profileQueryId = 1L, + brokerName = "TestBroker", + name = "John Michael Doe", + profileUrl = "https://example.com/profile", + identifier = "identifier-123", + addresses = addresses, + extras = extras, + dateAddedInMillis = dateAddedInMillis, + deprecated = deprecated, + ) + + private fun nonDeprecatedUserProfile(profileQueryId: Long) = UserProfile( + id = profileQueryId, + userName = UserName(firstName = "John", lastName = "Doe"), + addresses = com.duckduckgo.pir.impl.store.db.Address(city = "NYC", state = "NY"), + birthYear = 1990, + deprecated = false, + ) + + private fun captureInserted(): List = saveArgumentCaptors().first + + private fun captureRefreshed(): List = saveArgumentCaptors().second + + private fun saveArgumentCaptors(): Pair, List> { + val insertCaptor = argumentCaptor>() + val refreshCaptor = argumentCaptor>() + verify(mockExtractedProfileDao).insertAndRefreshExtractedProfiles(insertCaptor.capture(), refreshCaptor.capture()) + return insertCaptor.firstValue to refreshCaptor.firstValue } @Test