From bfb135c2a6e98a0d37df753f39e46157cf70f0fd Mon Sep 17 00:00:00 2001 From: John Hayes Date: Mon, 20 Jul 2026 16:44:52 +0100 Subject: [PATCH 1/4] Add Microsoft Purview plugin v1.4.3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add dataSourceName field to Scans on Data Source stream, read directly from the API response rather than parsed from the scan id - Fix lastRunResult fields in dataSourceScans and scansForDataSource: status was scanRunStatus, times were nested under discoveryExecutionDetails, and runType was scanLevelType — all previously broken due to incorrect field name assumptions Co-Authored-By: Claude Sonnet 4.6 --- .../MicrosoftPurview/v1/configValidation.json | 18 ++ plugins/MicrosoftPurview/v1/custom_types.json | 16 ++ .../v1/dataStreams/accountAssetCount.json | 25 ++ .../dataStreams/assetCountByEntityType.json | 37 +++ .../v1/dataStreams/assetLineage.json | 72 +++++ .../v1/dataStreams/assetsByCollection.json | 42 +++ .../dataStreams/assetsByEntityTypeGlobal.json | 26 ++ .../dataStreams/classificationCoverage.json | 31 ++ .../v1/dataStreams/classificationTypes.json | 27 ++ .../collectionClassificationSummary.json | 36 +++ .../dataStreams/collectionPolicyCoverage.json | 34 +++ .../v1/dataStreams/collections.json | 72 +++++ .../v1/dataStreams/dataSourceScans.json | 75 +++++ .../v1/dataStreams/listDataSources.json | 82 ++++++ .../v1/dataStreams/metadataPolicies.json | 27 ++ .../v1/dataStreams/scanHistory.json | 85 ++++++ .../v1/dataStreams/scansForDataSource.json | 34 +++ .../dataStreams/scripts/accountAssetCount.js | 2 + .../scripts/assetCountByEntityType.js | 2 + .../v1/dataStreams/scripts/assetLineage.js | 27 ++ .../dataStreams/scripts/assetsByCollection.js | 9 + .../scripts/assetsByEntityTypeGlobal.js | 8 + .../scripts/classificationCoverage.js | 14 + .../scripts/classificationTypes.js | 6 + .../collectionClassificationSummary.js | 9 + .../scripts/collectionPolicyCoverage.js | 29 ++ .../v1/dataStreams/scripts/dataSourceScans.js | 21 ++ .../dataStreams/scripts/metadataPolicies.js | 11 + .../v1/dataStreams/scripts/searchAssets.js | 8 + .../v1/dataStreams/searchAssets.json | 58 ++++ .../collectionDashboard.dash.json | 196 +++++++++++++ .../dataSourceDashboard.dash.json | 115 ++++++++ .../v1/defaultContent/manifest.json | 7 + .../v1/defaultContent/overview.dash.json | 267 ++++++++++++++++++ .../v1/defaultContent/scopes.json | 32 +++ plugins/MicrosoftPurview/v1/docs/README.md | 66 +++++ plugins/MicrosoftPurview/v1/icon.svg | 30 ++ .../v1/indexDefinitions/default.json | 36 +++ plugins/MicrosoftPurview/v1/metadata.json | 70 +++++ plugins/MicrosoftPurview/v1/ui.json | 39 +++ 40 files changed, 1801 insertions(+) create mode 100644 plugins/MicrosoftPurview/v1/configValidation.json create mode 100644 plugins/MicrosoftPurview/v1/custom_types.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/accountAssetCount.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/assetCountByEntityType.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/assetLineage.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/assetsByCollection.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/assetsByEntityTypeGlobal.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/classificationCoverage.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/classificationTypes.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/collectionClassificationSummary.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/collectionPolicyCoverage.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/collections.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/dataSourceScans.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/listDataSources.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/metadataPolicies.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scanHistory.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scansForDataSource.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/accountAssetCount.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/assetCountByEntityType.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/assetLineage.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/assetsByCollection.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/assetsByEntityTypeGlobal.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/classificationCoverage.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/classificationTypes.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionClassificationSummary.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionPolicyCoverage.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/dataSourceScans.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/metadataPolicies.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/searchAssets.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/searchAssets.json create mode 100644 plugins/MicrosoftPurview/v1/defaultContent/collectionDashboard.dash.json create mode 100644 plugins/MicrosoftPurview/v1/defaultContent/dataSourceDashboard.dash.json create mode 100644 plugins/MicrosoftPurview/v1/defaultContent/manifest.json create mode 100644 plugins/MicrosoftPurview/v1/defaultContent/overview.dash.json create mode 100644 plugins/MicrosoftPurview/v1/defaultContent/scopes.json create mode 100644 plugins/MicrosoftPurview/v1/docs/README.md create mode 100644 plugins/MicrosoftPurview/v1/icon.svg create mode 100644 plugins/MicrosoftPurview/v1/indexDefinitions/default.json create mode 100644 plugins/MicrosoftPurview/v1/metadata.json create mode 100644 plugins/MicrosoftPurview/v1/ui.json diff --git a/plugins/MicrosoftPurview/v1/configValidation.json b/plugins/MicrosoftPurview/v1/configValidation.json new file mode 100644 index 00000000..e4c22a98 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/configValidation.json @@ -0,0 +1,18 @@ +{ + "steps": [ + { + "displayName": "Authenticate and list collections", + "dataStream": { "name": "collections" }, + "required": true, + "error": "Could not authenticate or read collections. Check the account name, tenant/client ID, secret, and that the app registration has been added as Data Reader on the Purview root collection.", + "success": "Connected to Purview and read collections." + }, + { + "displayName": "List data sources", + "dataStream": { "name": "listDataSources" }, + "required": false, + "error": "Authenticated but could not list data sources. Add the app registration as Data Source Administrator on the Purview root collection.", + "success": "Data sources accessible." + } + ] +} diff --git a/plugins/MicrosoftPurview/v1/custom_types.json b/plugins/MicrosoftPurview/v1/custom_types.json new file mode 100644 index 00000000..09941497 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/custom_types.json @@ -0,0 +1,16 @@ +[ + { + "name": "Purview Collection", + "sourceType": "Purview Collection", + "icon": "folder-tree", + "singular": "Collection", + "plural": "Collections" + }, + { + "name": "Purview Data Source", + "sourceType": "Purview Data Source", + "icon": "database", + "singular": "Data Source", + "plural": "Data Sources" + } +] diff --git a/plugins/MicrosoftPurview/v1/dataStreams/accountAssetCount.json b/plugins/MicrosoftPurview/v1/dataStreams/accountAssetCount.json new file mode 100644 index 00000000..dfd27ff7 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/accountAssetCount.json @@ -0,0 +1,25 @@ +{ + "name": "accountAssetCount", + "displayName": "Total Asset Count", + "description": "Total number of assets cataloged across the whole Purview account", + "tags": ["Assets"], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "post", + "endpointPath": "datamap/api/search/query", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "postBody": { + "keywords": null, + "limit": 1 + }, + "postRequestScript": "accountAssetCount.js" + }, + "matches": "none", + "metadata": [ + { "name": "label", "displayName": "Label", "role": "label" }, + { "name": "totalAssets", "displayName": "Total Assets", "shape": "number", "role": "value" } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/assetCountByEntityType.json b/plugins/MicrosoftPurview/v1/dataStreams/assetCountByEntityType.json new file mode 100644 index 00000000..ab1f3016 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/assetCountByEntityType.json @@ -0,0 +1,37 @@ +{ + "name": "assetCountByEntityType", + "displayName": "Asset Count by Entity Type", + "description": "Count of assets in the selected collection grouped by entity type", + "tags": ["Assets"], + "baseDataSourceName": "httpRequestScopedSingle", + "config": { + "httpMethod": "post", + "endpointPath": "datamap/api/search/query", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "postBody": { + "keywords": null, + "limit": 1, + "filter": { + "collectionId": "{{object.collectionName}}" + }, + "facets": [ + { + "facet": "entityType", + "count": 50, + "sort": { "count": "desc" } + } + ] + }, + "pathToData": "@search.facets.entityType", + "postRequestScript": "assetCountByEntityType.js" + }, + "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Collection"] } }, + "metadata": [ + { "name": "entityType", "displayName": "Entity Type", "role": "label" }, + { "name": "count", "displayName": "Assets", "shape": "number" } + ], + "timeframes": false, + "defaultShaping": { "sort": { "by": [["count", "desc"]] } } +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/assetLineage.json b/plugins/MicrosoftPurview/v1/dataStreams/assetLineage.json new file mode 100644 index 00000000..6dd965e4 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/assetLineage.json @@ -0,0 +1,72 @@ +{ + "name": "assetLineage", + "displayName": "Asset Lineage", + "description": "Upstream and downstream lineage for an asset identified by its GUID", + "tags": ["Assets"], + "baseDataSourceName": "httpRequestUnscoped", + "ui": [ + { + "type": "text", + "name": "assetGuid", + "label": "Asset GUID", + "placeholder": "Asset GUID from the catalog", + "validation": { "required": true }, + "help": "The Purview asset's GUID. Copy it from the asset details page in the Purview portal, or from the **Asset GUID** column of the Search Assets stream." + }, + { + "type": "switch", + "name": "direction", + "label": "Direction", + "defaultValue": "BOTH", + "options": [ + { "value": "BOTH", "label": "Both" }, + { "value": "INPUT", "label": "Upstream" }, + { "value": "OUTPUT", "label": "Downstream" } + ] + }, + { + "type": "number", + "name": "depth", + "label": "Depth", + "defaultValue": 3, + "help": "Number of hops to traverse in each direction." + } + ], + "config": { + "httpMethod": "get", + "endpointPath": "datamap/api/atlas/v2/lineage/{{assetGuid}}", + "getArgs": [ + { "key": "direction", "value": "{{direction}}" }, + { "key": "depth", "value": "{{depth}}" } + ], + "postRequestScript": "assetLineage.js" + }, + "manualConfigApply": true, + "matches": "none", + "metadata": [ + { "name": "guid", "displayName": "GUID", "shape": "guid", "visible": false }, + { "name": "name", "displayName": "Name", "role": "label" }, + { "name": "typeName", "displayName": "Entity Type" }, + { "name": "qualifiedName", "displayName": "Qualified Name" }, + { + "name": "role", + "displayName": "Role", + "shape": [ + "state", + { + "map": { + "success": ["Base"], + "warning": ["Upstream"], + "unknown": ["Downstream"] + } + } + ] + }, + { "name": "status", "displayName": "Status" }, + { + "name": "classifications", + "displayName": "Classifications" + } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/assetsByCollection.json b/plugins/MicrosoftPurview/v1/dataStreams/assetsByCollection.json new file mode 100644 index 00000000..e0948dff --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/assetsByCollection.json @@ -0,0 +1,42 @@ +{ + "name": "assetsByCollection", + "displayName": "Assets in Collection", + "description": "Assets found in the selected Purview collection", + "tags": ["Assets"], + "baseDataSourceName": "httpRequestScopedSingle", + "config": { + "httpMethod": "post", + "endpointPath": "datamap/api/search/query", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "postBody": { + "keywords": null, + "limit": 100, + "filter": { + "collectionId": "{{object.collectionName}}" + }, + "orderby": [{ "name": "ASC" }] + }, + "postRequestScript": "assetsByCollection.js", + "paging": { + "mode": "token", + "in": { "realm": "payload", "path": "continuationToken" }, + "out": { "realm": "body", "path": "continuationToken" } + } + }, + "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Collection"] } }, + "metadata": [ + { "name": "id", "displayName": "Asset GUID", "shape": "guid", "visible": false }, + { "name": "name", "displayName": "Name", "role": "label" }, + { "name": "qualifiedName", "displayName": "Qualified Name" }, + { "name": "entityType", "displayName": "Entity Type" }, + { "name": "assetType", "displayName": "Asset Type" }, + { "name": "classification", "displayName": "Classifications" }, + { "name": "term", "displayName": "Glossary Terms" }, + { "name": "label", "displayName": "Labels" }, + { "name": "description", "displayName": "Description" }, + { "name": "owner", "displayName": "Owner" } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/assetsByEntityTypeGlobal.json b/plugins/MicrosoftPurview/v1/dataStreams/assetsByEntityTypeGlobal.json new file mode 100644 index 00000000..0d52ae2c --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/assetsByEntityTypeGlobal.json @@ -0,0 +1,26 @@ +{ + "name": "assetsByEntityTypeGlobal", + "displayName": "Asset Count by Entity Type", + "description": "Count of assets grouped by entity type across the whole account", + "tags": ["Assets"], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "post", + "endpointPath": "datamap/api/search/query", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "postBody": { + "keywords": null, + "limit": 1000 + }, + "postRequestScript": "assetsByEntityTypeGlobal.js" + }, + "matches": "none", + "metadata": [ + { "name": "entityType", "displayName": "Entity Type", "role": "label" }, + { "name": "count", "displayName": "Assets", "shape": "number" } + ], + "timeframes": false, + "defaultShaping": { "sort": { "by": [["count", "desc"]] } } +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/classificationCoverage.json b/plugins/MicrosoftPurview/v1/dataStreams/classificationCoverage.json new file mode 100644 index 00000000..d2a19f6a --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/classificationCoverage.json @@ -0,0 +1,31 @@ +{ + "name": "classificationCoverage", + "displayName": "Classification Coverage", + "description": "Count of assets carrying each classification across the whole account", + "tags": ["Classifications"], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "post", + "endpointPath": "datamap/api/search/query", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "postBody": { + "keywords": null, + "limit": 1000 + }, + "postRequestScript": "classificationCoverage.js" + }, + "matches": "none", + "metadata": [ + { "name": "classification", "displayName": "Classification", "role": "label" }, + { "name": "assets", "displayName": "Assets", "shape": "number" }, + { + "name": "coverage", + "displayName": "% of Total Assets", + "shape": ["percent", { "asZeroToOne": true, "decimalPlaces": 1 }] + } + ], + "timeframes": false, + "defaultShaping": { "sort": { "by": [["assets", "desc"]] } } +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/classificationTypes.json b/plugins/MicrosoftPurview/v1/dataStreams/classificationTypes.json new file mode 100644 index 00000000..ec4d4b13 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/classificationTypes.json @@ -0,0 +1,27 @@ +{ + "name": "classificationTypes", + "displayName": "Classification Types", + "description": "All classification type definitions in the Purview account", + "tags": ["Classifications"], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "get", + "endpointPath": "datamap/api/atlas/v2/types/typedefs", + "getArgs": [ + { "key": "type", "value": "classification" } + ], + "postRequestScript": "classificationTypes.js" + }, + "matches": "none", + "metadata": [ + { "name": "name", "displayName": "Classification", "role": "label" }, + { "name": "description", "displayName": "Description" }, + { "name": "category", "displayName": "Category" }, + { "name": "typeVersion", "displayName": "Version" }, + { "name": "superTypes", "displayName": "Super Types" }, + { "name": "createdBy", "displayName": "Created By" }, + { "name": "createTime", "displayName": "Created", "shape": "date" }, + { "name": "guid", "displayName": "GUID", "shape": "guid", "visible": false } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/collectionClassificationSummary.json b/plugins/MicrosoftPurview/v1/dataStreams/collectionClassificationSummary.json new file mode 100644 index 00000000..92d91c02 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/collectionClassificationSummary.json @@ -0,0 +1,36 @@ +{ + "name": "collectionClassificationSummary", + "displayName": "Classification Summary", + "description": "Classified vs unclassified asset counts for the selected collection", + "tags": ["Classifications"], + "baseDataSourceName": "httpRequestScopedSingle", + "config": { + "httpMethod": "post", + "endpointPath": "datamap/api/search/query", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "postBody": { + "keywords": null, + "limit": 1, + "filter": { + "collectionId": "{{object.collectionName}}" + }, + "facets": [ + { + "facet": "classification", + "count": 200, + "sort": { "count": "desc" } + } + ] + }, + "postRequestScript": "collectionClassificationSummary.js" + }, + "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Collection"] } }, + "metadata": [ + { "name": "state", "displayName": "State", "shape": ["state", { "map": { "success": ["Classified"], "warning": ["Unclassified"] } }] }, + { "name": "label", "displayName": "Category", "role": "label" }, + { "name": "assets", "displayName": "Assets", "shape": "number", "role": "value" } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/collectionPolicyCoverage.json b/plugins/MicrosoftPurview/v1/dataStreams/collectionPolicyCoverage.json new file mode 100644 index 00000000..89cc9251 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/collectionPolicyCoverage.json @@ -0,0 +1,34 @@ +{ + "name": "collectionPolicyCoverage", + "displayName": "Policy Coverage", + "description": "Metadata policies attached to the selected collection", + "tags": ["Policies"], + "baseDataSourceName": "httpRequestScopedSingle", + "config": { + "httpMethod": "get", + "endpointPath": "policyStore/metadataPolicies", + "getArgs": [ + { "key": "api-version", "value": "2021-07-01-preview" }, + { "key": "collectionName", "value": "{{object.collectionName}}" } + ], + "postRequestScript": "collectionPolicyCoverage.js" + }, + "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Collection"] } }, + "metadata": [ + { "name": "id", "displayName": "Policy ID", "shape": "guid", "visible": false }, + { "name": "name", "displayName": "Policy", "role": "label" }, + { "name": "version", "displayName": "Version", "shape": "number" }, + { "name": "description", "displayName": "Description" }, + { "name": "decisionRuleCount", "displayName": "Decision Rules", "shape": "number" }, + { "name": "attributeRuleCount", "displayName": "Attribute Rules", "shape": "number" }, + { + "name": "state", + "displayName": "State", + "shape": [ + "state", + { "map": { "success": ["Attached"], "warning": ["No policy"] } } + ] + } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/collections.json b/plugins/MicrosoftPurview/v1/dataStreams/collections.json new file mode 100644 index 00000000..af86ac55 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/collections.json @@ -0,0 +1,72 @@ +{ + "name": "collections", + "displayName": "Collections", + "description": "Collections (organizational units) in the Purview account", + "tags": ["Collections"], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "get", + "endpointPath": "account/collections", + "getArgs": [ + { "key": "api-version", "value": "2019-11-01-preview" } + ], + "pathToData": "value", + "paging": { + "mode": "nextUrl", + "in": { "realm": "payload", "path": "nextLink" } + } + }, + "matches": "none", + "metadata": [ + { + "name": "name", + "displayName": "Collection ID", + "role": "sourceId", + "visible": false + }, + { + "name": "collectionName", + "displayName": "Collection", + "computed": true, + "valueExpression": "{{ $['name'] }}", + "visible": false + }, + { + "name": "friendlyName", + "displayName": "Friendly Name", + "visible": false + }, + { + "name": "displayName", + "displayName": "Name", + "computed": true, + "valueExpression": "{{ $['friendlyName'] || $['name'] }}", + "role": "label" + }, + { + "name": "description", + "displayName": "Description" + }, + { + "name": "parentCollection", + "visible": false + }, + { + "name": "parentCollectionName", + "computed": true, + "valueExpression": "{{ $['parentCollection'] && $['parentCollection'].referenceName ? $['parentCollection'].referenceName : '' }}", + "displayName": "Parent Collection" + }, + { + "name": "collectionProvisioningState", + "displayName": "State" + }, + { + "name": "sourceType", + "computed": true, + "valueExpression": "Purview Collection", + "visible": false + } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/dataSourceScans.json b/plugins/MicrosoftPurview/v1/dataStreams/dataSourceScans.json new file mode 100644 index 00000000..c0d0a906 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/dataSourceScans.json @@ -0,0 +1,75 @@ +{ + "name": "dataSourceScans", + "displayName": "Scans on Data Source", + "description": "Scans configured on the selected data source with latest run status", + "tags": ["Data Sources"], + "baseDataSourceName": "httpRequestScopedSingle", + "config": { + "httpMethod": "get", + "endpointPath": "scan/datasources/{{object.rawId}}/scans", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "postRequestScript": "dataSourceScans.js", + "paging": { + "mode": "nextUrl", + "in": { "realm": "payload", "path": "nextLink" } + } + }, + "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Data Source"] } }, + "metadata": [ + { "name": "name", "displayName": "Scan", "role": "label" }, + { "name": "dataSourceName", "displayName": "Data Source" }, + { "name": "kind", "displayName": "Kind" }, + { + "name": "properties.collection.referenceName", + "displayName": "Collection", + "visible": false + }, + { + "name": "collection", + "computed": true, + "valueExpression": "{{ $['properties.collection.referenceName'] || '' }}", + "displayName": "Collection" + }, + { + "name": "properties.scanRulesetName", + "displayName": "Ruleset" + }, + { + "name": "properties.scanRulesetType", + "displayName": "Ruleset Type" + }, + { + "name": "lastRunResult.status", + "displayName": "Last Status", + "shape": [ + "state", + { + "map": { + "success": ["Succeeded"], + "error": ["Failed", "TransientFailure"], + "warning": ["Canceled"], + "unknown": ["Accepted", "InProgress"] + } + } + ] + }, + { + "name": "lastRunResult.startTime", + "displayName": "Last Run Started", + "shape": "date" + }, + { + "name": "lastRunResult.endTime", + "displayName": "Last Run Ended", + "shape": "date" + }, + { + "name": "lastRunResult.scanLevelType", + "displayName": "Scan Level" + }, + { "name": "creationType", "displayName": "Creation Type" } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/listDataSources.json b/plugins/MicrosoftPurview/v1/dataStreams/listDataSources.json new file mode 100644 index 00000000..5725eb62 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/listDataSources.json @@ -0,0 +1,82 @@ +{ + "name": "listDataSources", + "displayName": "Data Sources", + "description": "Registered data sources in the Purview account", + "tags": ["Data Sources"], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "get", + "endpointPath": "scan/datasources", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "pathToData": "value", + "expandInnerObjects": true, + "paging": { + "mode": "nextUrl", + "in": { "realm": "payload", "path": "nextLink" } + } + }, + "matches": "none", + "metadata": [ + { "name": "name", "displayName": "Name", "role": "label" }, + { "name": "kind", "displayName": "Kind" }, + { + "name": "properties.collection.referenceName", + "displayName": "Collection", + "visible": false + }, + { + "name": "collection", + "computed": true, + "valueExpression": "{{ $['properties.collection.referenceName'] || '' }}", + "displayName": "Collection" + }, + { + "name": "properties.resourceGroup", + "displayName": "Resource Group", + "visible": false + }, + { + "name": "resourceGroup", + "computed": true, + "valueExpression": "{{ $['properties.resourceGroup'] || '' }}", + "displayName": "Resource Group" + }, + { + "name": "properties.subscriptionId", + "displayName": "Subscription", + "visible": false + }, + { + "name": "subscriptionId", + "computed": true, + "valueExpression": "{{ $['properties.subscriptionId'] || '' }}", + "displayName": "Subscription", + "visible": false + }, + { + "name": "subscriptionUrl", + "computed": true, + "valueExpression": "{{ $['properties.subscriptionId'] ? 'https://portal.azure.com/#resource/subscriptions/' + $['properties.subscriptionId'] : '' }}", + "displayName": "Subscription", + "shape": "url" + }, + { + "name": "properties.createdAt", + "displayName": "Created", + "shape": "date" + }, + { + "name": "creationType", + "displayName": "Creation Type" + }, + { + "name": "sourceType", + "computed": true, + "valueExpression": "Purview Data Source", + "visible": false + } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/metadataPolicies.json b/plugins/MicrosoftPurview/v1/dataStreams/metadataPolicies.json new file mode 100644 index 00000000..8e7f22e0 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/metadataPolicies.json @@ -0,0 +1,27 @@ +{ + "name": "metadataPolicies", + "displayName": "Metadata Policies", + "description": "All metadata policies across the Purview account", + "tags": ["Policies"], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "get", + "endpointPath": "policyStore/metadataPolicies", + "getArgs": [ + { "key": "api-version", "value": "2021-07-01-preview" } + ], + "postRequestScript": "metadataPolicies.js" + }, + "matches": "none", + "metadata": [ + { "name": "id", "displayName": "Policy ID", "shape": "guid", "visible": false }, + { "name": "name", "displayName": "Name", "role": "label" }, + { "name": "version", "displayName": "Version", "shape": "number" }, + { "name": "collection", "displayName": "Collection" }, + { "name": "description", "displayName": "Description" }, + { "name": "decisionRuleCount", "displayName": "Decision Rules", "shape": "number" }, + { "name": "attributeRuleCount", "displayName": "Attribute Rules", "shape": "number" } + ], + "timeframes": false, + "defaultShaping": { "sort": { "by": [["collection", "asc"]] } } +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scanHistory.json b/plugins/MicrosoftPurview/v1/dataStreams/scanHistory.json new file mode 100644 index 00000000..d5d200e0 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scanHistory.json @@ -0,0 +1,85 @@ +{ + "name": "scanHistory", + "displayName": "Scan History", + "description": "Historical scan runs for a named scan on the selected data source", + "tags": ["Data Sources"], + "baseDataSourceName": "httpRequestScopedSingle", + "ui": [ + { + "type": "autocomplete", + "name": "scanName", + "label": "Scan Name", + "placeholder": "Select a scan", + "isMulti": false, + "allowCustomValues": false, + "validation": { "required": true }, + "data": { + "source": "dataStream", + "dataStreamName": "scansForDataSource", + "dataSourceConfig": {} + } + } + ], + "config": { + "httpMethod": "get", + "endpointPath": "scan/datasources/{{object.rawId}}/scans/{{scanName}}/runs", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "pathToData": "value", + "expandInnerObjects": true, + "paging": { + "mode": "nextUrl", + "in": { "realm": "payload", "path": "nextLink" } + } + }, + "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Data Source"] } }, + "metadata": [ + { "name": "id", "displayName": "Run ID", "shape": "guid", "visible": false }, + { "name": "startTime", "displayName": "Started", "shape": "date", "role": "timestamp" }, + { "name": "endTime", "displayName": "Ended", "shape": "date" }, + { + "name": "status", + "displayName": "Status", + "shape": [ + "state", + { + "map": { + "success": ["Succeeded"], + "error": ["Failed", "TransientFailure"], + "warning": ["Canceled"], + "unknown": ["Accepted", "InProgress"] + } + } + ] + }, + { "name": "runType", "displayName": "Run Type" }, + { "name": "scanLevelType", "displayName": "Scan Level" }, + { + "name": "duration", + "displayName": "Duration", + "computed": true, + "valueExpression": "{{ $['startTime'] && $['endTime'] ? (new Date($['endTime']).getTime() - new Date($['startTime']).getTime()) / 1000 : null }}", + "shape": ["seconds", { "formatDuration": true, "decimalPlaces": 0 }] + }, + { + "name": "discoveryExecutionDetails.statistics.assets.discovered", + "displayName": "Assets Discovered", + "shape": "number" + }, + { + "name": "discoveryExecutionDetails.statistics.assets.classified", + "displayName": "Assets Classified", + "shape": "number" + }, + { + "name": "discoveryExecutionDetails.statistics.assets.failed", + "displayName": "Assets Failed", + "shape": "number" + }, + { "name": "errorMessage", "displayName": "Error", "visible": false }, + { "name": "dataSourceType", "displayName": "Data Source Type" } + ], + "timeframes": true, + "defaultShaping": { "sort": { "by": [["startTime", "desc"]] } } +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scansForDataSource.json b/plugins/MicrosoftPurview/v1/dataStreams/scansForDataSource.json new file mode 100644 index 00000000..647c7fd7 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scansForDataSource.json @@ -0,0 +1,34 @@ +{ + "name": "scansForDataSource", + "displayName": "Scan Definitions", + "description": "Lists the scan definitions configured on a data source, with their last run date, sorted most-recent-first. Used to populate the Scan Name picker on the Scan History stream.", + "baseDataSourceName": "httpRequestScopedSingle", + "visibility": { "type": "hidden" }, + "config": { + "httpMethod": "get", + "endpointPath": "scan/datasources/{{object.rawId}}/scans", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "pathToData": "value", + "expandInnerObjects": true, + "paging": { + "mode": "nextUrl", + "in": { "realm": "payload", "path": "nextLink" } + } + }, + "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Data Source"] } }, + "metadata": [ + { "name": "name", "displayName": "Scan Name", "role": "value", "visible": false }, + { "name": "lastRunResult.discoveryExecutionDetails.discoveryStartTime", "displayName": "Last Run Started", "shape": "date", "visible": false }, + { + "name": "scanLabel", + "displayName": "Scan", + "role": "label", + "computed": true, + "valueExpression": "{{ $['lastRunResult.discoveryExecutionDetails.discoveryStartTime'] ? $['name'] + ' — ' + new Date($['lastRunResult.discoveryExecutionDetails.discoveryStartTime']).toISOString().slice(0, 10) : $['name'] + ' — never run' }}" + } + ], + "timeframes": false, + "defaultShaping": { "sort": { "by": [["lastRunResult.discoveryExecutionDetails.discoveryStartTime", "desc"]] } } +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/accountAssetCount.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/accountAssetCount.js new file mode 100644 index 00000000..b0a26dc7 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/accountAssetCount.js @@ -0,0 +1,2 @@ +const total = Number(data?.['@search.count']) || 0; +result = [{ label: 'Total Assets', totalAssets: total }]; diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetCountByEntityType.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetCountByEntityType.js new file mode 100644 index 00000000..4217db2e --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetCountByEntityType.js @@ -0,0 +1,2 @@ +const facets = data?.['@search.facets']?.entityType || []; +result = facets.map((f) => ({ entityType: f.value, count: Number(f.count) })); diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetLineage.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetLineage.js new file mode 100644 index 00000000..235589fb --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetLineage.js @@ -0,0 +1,27 @@ +const baseGuid = data?.baseEntityGuid; +const map = data?.guidEntityMap || {}; +const relations = data?.relations || []; + +const upstream = new Set(); +const downstream = new Set(); +for (const r of relations) { + if (r.toEntityId === baseGuid) upstream.add(r.fromEntityId); + if (r.fromEntityId === baseGuid) downstream.add(r.toEntityId); +} + +result = Object.entries(map).map(([guid, e]) => { + let role = 'Downstream'; + if (guid === baseGuid) role = 'Base'; + else if (upstream.has(guid)) role = 'Upstream'; + + const attrs = e.attributes || {}; + return { + guid, + name: attrs.name || e.displayText || guid, + typeName: e.typeName || '', + qualifiedName: attrs.qualifiedName || '', + role, + status: e.status || '', + classifications: (e.classificationNames || []).join(', '), + }; +}); diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetsByCollection.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetsByCollection.js new file mode 100644 index 00000000..9fcac4e9 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetsByCollection.js @@ -0,0 +1,9 @@ +const arrayToText = (v) => Array.isArray(v) ? v.join(', ') : (v == null ? '' : String(v)); + +result = (data?.value || []).map((hit) => ({ + ...hit, + assetType: arrayToText(hit.assetType), + classification: arrayToText(hit.classification), + term: arrayToText(hit.term), + label: arrayToText(hit.label) +})); diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetsByEntityTypeGlobal.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetsByEntityTypeGlobal.js new file mode 100644 index 00000000..835db28c --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetsByEntityTypeGlobal.js @@ -0,0 +1,8 @@ +const counts = {}; +for (const item of (data?.value || [])) { + const et = item.entityType || 'unknown'; + counts[et] = (counts[et] || 0) + 1; +} +result = Object.entries(counts) + .map(([entityType, count]) => ({ entityType, count })) + .sort((a, b) => b.count - a.count); diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/classificationCoverage.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/classificationCoverage.js new file mode 100644 index 00000000..2ae69013 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/classificationCoverage.js @@ -0,0 +1,14 @@ +const total = Number(data?.['@search.count']) || 0; +const counts = {}; +for (const item of (data?.value || [])) { + for (const c of (Array.isArray(item.classification) ? item.classification : [])) { + counts[c] = (counts[c] || 0) + 1; + } +} +result = Object.entries(counts) + .map(([classification, assets]) => ({ + classification, + assets, + coverage: total > 0 ? assets / total : 0 + })) + .sort((a, b) => b.assets - a.assets); diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/classificationTypes.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/classificationTypes.js new file mode 100644 index 00000000..2cb80236 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/classificationTypes.js @@ -0,0 +1,6 @@ +const arrayToText = (v) => Array.isArray(v) ? v.join(', ') : (v == null ? '' : String(v)); + +result = (data?.classificationDefs || []).map((def) => ({ + ...def, + superTypes: arrayToText(def.superTypes) +})); diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionClassificationSummary.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionClassificationSummary.js new file mode 100644 index 00000000..82daace8 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionClassificationSummary.js @@ -0,0 +1,9 @@ +const total = Number(data?.['@search.count']) || 0; +const facets = data?.['@search.facets']?.classification || []; +const classified = facets.reduce((sum, f) => sum + (Number(f.count) || 0), 0); +const unclassified = Math.max(0, total - classified); + +result = [ + { state: 'Classified', label: 'Classified', assets: classified }, + { state: 'Unclassified', label: 'Unclassified', assets: unclassified }, +]; diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionPolicyCoverage.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionPolicyCoverage.js new file mode 100644 index 00000000..c69f513f --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionPolicyCoverage.js @@ -0,0 +1,29 @@ +const policies = data?.values || data?.value || []; + +if (policies.length === 0) { + result = [ + { + id: null, + name: '(no policy)', + version: 0, + description: '', + decisionRuleCount: 0, + attributeRuleCount: 0, + state: 'No policy', + }, + ]; +} else { + result = policies.map((p) => ({ + id: p.id, + name: p.name, + version: Number(p.version) || 0, + description: p.properties?.description || '', + decisionRuleCount: Array.isArray(p.properties?.decisionRules) + ? p.properties.decisionRules.length + : 0, + attributeRuleCount: Array.isArray(p.properties?.attributeRules) + ? p.properties.attributeRules.length + : 0, + state: 'Attached', + })); +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/dataSourceScans.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/dataSourceScans.js new file mode 100644 index 00000000..97279c43 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/dataSourceScans.js @@ -0,0 +1,21 @@ +const scans = data?.value || []; + +result = scans.map((scan) => { + const lr = scan.lastRunResult || {}; + const discovery = lr.discoveryExecutionDetails || {}; + const props = scan.properties || {}; + + return { + name: scan.name || '', + dataSourceName: scan.dataSourceName || '', + kind: scan.kind || '', + creationType: scan.creationType || '', + 'properties.collection.referenceName': props.collection?.referenceName || '', + 'properties.scanRulesetName': props.scanRulesetName || '', + 'properties.scanRulesetType': props.scanRulesetType || '', + 'lastRunResult.status': lr.scanRunStatus || '', + 'lastRunResult.startTime': discovery.discoveryStartTime || null, + 'lastRunResult.endTime': discovery.discoveryEndTime || null, + 'lastRunResult.scanLevelType': lr.scanLevelType || '', + }; +}); diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/metadataPolicies.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/metadataPolicies.js new file mode 100644 index 00000000..cb06aa3c --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/metadataPolicies.js @@ -0,0 +1,11 @@ +const policies = data?.values || data?.value || []; + +result = policies.map((p) => ({ + id: p.id, + name: p.name, + version: Number(p.version) || 0, + collection: p.properties?.collection?.referenceName || '', + description: p.properties?.description || '', + decisionRuleCount: Array.isArray(p.properties?.decisionRules) ? p.properties.decisionRules.length : 0, + attributeRuleCount: Array.isArray(p.properties?.attributeRules) ? p.properties.attributeRules.length : 0, +})); diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/searchAssets.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/searchAssets.js new file mode 100644 index 00000000..271e8d5b --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/searchAssets.js @@ -0,0 +1,8 @@ +const arrayToText = (v) => Array.isArray(v) ? v.join(', ') : (v == null ? '' : String(v)); + +result = (data?.value || []).map((hit) => ({ + ...hit, + assetType: arrayToText(hit.assetType), + classification: arrayToText(hit.classification), + term: arrayToText(hit.term) +})); diff --git a/plugins/MicrosoftPurview/v1/dataStreams/searchAssets.json b/plugins/MicrosoftPurview/v1/dataStreams/searchAssets.json new file mode 100644 index 00000000..4beef9d6 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/searchAssets.json @@ -0,0 +1,58 @@ +{ + "name": "searchAssets", + "displayName": "Search Assets", + "description": "Search the Purview catalog with optional keyword, entity-type, and classification filters", + "tags": ["Assets"], + "baseDataSourceName": "httpRequestUnscoped", + "ui": [ + { + "type": "text", + "name": "keywords", + "label": "Keywords", + "placeholder": "Search keywords (optional)" + }, + { + "type": "text", + "name": "entityType", + "label": "Entity Type", + "placeholder": "e.g. azure_blob_path" + }, + { + "type": "text", + "name": "classification", + "label": "Classification", + "placeholder": "e.g. MICROSOFT.GOVERNMENT.US.SOCIAL_SECURITY_NUMBER" + }, + { + "type": "number", + "name": "limit", + "label": "Result Limit", + "defaultValue": 100 + } + ], + "config": { + "httpMethod": "post", + "endpointPath": "datamap/api/search/query", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "postBody": { + "keywords": "{{ keywords || null }}", + "limit": "{{ Number(limit) || 100 }}", + "filter": "{{ (entityType || classification) ? { and: [entityType ? { entityType: entityType } : null, classification ? { classification: classification } : null].filter(Boolean) } : null }}" + }, + "postRequestScript": "searchAssets.js" + }, + "manualConfigApply": true, + "matches": "none", + "metadata": [ + { "name": "id", "displayName": "Asset GUID", "shape": "guid", "visible": false }, + { "name": "name", "displayName": "Name", "role": "label" }, + { "name": "qualifiedName", "displayName": "Qualified Name" }, + { "name": "entityType", "displayName": "Entity Type" }, + { "name": "assetType", "displayName": "Asset Type" }, + { "name": "classification", "displayName": "Classifications" }, + { "name": "term", "displayName": "Glossary Terms" } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/defaultContent/collectionDashboard.dash.json b/plugins/MicrosoftPurview/v1/defaultContent/collectionDashboard.dash.json new file mode 100644 index 00000000..7c4c53e3 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/defaultContent/collectionDashboard.dash.json @@ -0,0 +1,196 @@ +{ + "name": "Collection", + "schemaVersion": "1.5", + "variables": ["{{variables.Collection}}"], + "dashboard": { + "_type": "layout/grid", + "columns": 4, + "version": 1, + "contents": [ + { + "static": false, + "w": 1, + "moved": false, + "h": 4, + "x": 0, + "y": 0, + "i": "25f69cbd-2d27-4175-8273-75f70bea420d", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Properties", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "variables": ["{{variables.Collection}}"], + "dataStream": { "id": "datastream-properties" }, + "scope": { + "scope": "{{scopes.Collections}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.Collection}}" + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { "transpose": true } + } + } + } + }, + { + "static": false, + "w": 3, + "moved": false, + "h": 4, + "x": 1, + "y": 0, + "i": "5c456e3c-637a-4aea-b0a6-318dbbf4641e", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Classification Summary", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "variables": ["{{variables.Collection}}"], + "dataStream": { + "id": "{{dataStreams.collectionClassificationSummary}}", + "name": "collectionClassificationSummary", + "pluginConfigId": "{{configId}}" + }, + "scope": { + "scope": "{{scopes.Collections}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.Collection}}" + }, + "visualisation": { + "type": "data-stream-donut-chart", + "config": { + "data-stream-donut-chart": { + "valueColumn": "assets", + "labelColumn": "label", + "showValuesAsPercentage": true, + "legendPosition": "auto", + "legendMode": "table" + } + } + } + } + }, + { + "static": false, + "w": 2, + "moved": false, + "h": 5, + "x": 0, + "y": 4, + "i": "3b4de95e-0982-4058-8965-d362452e4150", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Asset Count by Entity Type", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "variables": ["{{variables.Collection}}"], + "dataStream": { + "id": "{{dataStreams.assetCountByEntityType}}", + "name": "assetCountByEntityType", + "pluginConfigId": "{{configId}}" + }, + "scope": { + "scope": "{{scopes.Collections}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.Collection}}" + }, + "visualisation": { + "type": "data-stream-bar-chart", + "config": { + "data-stream-bar-chart": { + "xAxisData": "entityType", + "yAxisData": ["count"], + "xAxisGroup": "none", + "showLegend": false, + "showGrid": true, + "horizontalLayout": "horizontal", + "displayMode": "actual", + "showValue": true, + "grouping": false, + "range": { "type": "auto" } + } + } + } + } + }, + { + "static": false, + "w": 2, + "moved": false, + "h": 5, + "x": 2, + "y": 4, + "i": "f335dba3-f381-4154-89cd-46674611bc64", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Policy Coverage", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "variables": ["{{variables.Collection}}"], + "dataStream": { + "id": "{{dataStreams.collectionPolicyCoverage}}", + "name": "collectionPolicyCoverage", + "pluginConfigId": "{{configId}}" + }, + "scope": { + "scope": "{{scopes.Collections}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.Collection}}" + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": false, + "columnOrder": ["state", "name", "version", "decisionRuleCount", "attributeRuleCount"] + } + } + } + } + }, + { + "static": false, + "w": 4, + "moved": false, + "h": 5, + "x": 0, + "y": 9, + "i": "1f931a39-24c6-416c-b564-75bb490bcf5c", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Assets in Collection", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "variables": ["{{variables.Collection}}"], + "dataStream": { + "id": "{{dataStreams.assetsByCollection}}", + "name": "assetsByCollection", + "pluginConfigId": "{{configId}}" + }, + "scope": { + "scope": "{{scopes.Collections}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.Collection}}" + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": false, + "columnOrder": ["name", "entityType", "assetType", "classification", "term", "qualifiedName"] + } + } + } + } + } + ] + } +} diff --git a/plugins/MicrosoftPurview/v1/defaultContent/dataSourceDashboard.dash.json b/plugins/MicrosoftPurview/v1/defaultContent/dataSourceDashboard.dash.json new file mode 100644 index 00000000..5f944db3 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/defaultContent/dataSourceDashboard.dash.json @@ -0,0 +1,115 @@ +{ + "name": "Data Source", + "schemaVersion": "1.5", + "variables": ["{{variables.DataSource}}"], + "dashboard": { + "_type": "layout/grid", + "columns": 4, + "version": 1, + "contents": [ + { + "static": false, + "w": 1, + "moved": false, + "h": 4, + "x": 0, + "y": 0, + "i": "0f4d8de5-c60c-4fa7-beb7-76260f18e504", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Properties", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "variables": ["{{variables.DataSource}}"], + "dataStream": { "id": "datastream-properties" }, + "scope": { + "scope": "{{scopes.DataSources}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.DataSource}}" + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { "transpose": true } + } + } + } + }, + { + "static": false, + "w": 3, + "moved": false, + "h": 4, + "x": 1, + "y": 0, + "i": "07d8ec4e-404c-4925-94c1-01a1da2e9938", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Scans", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "variables": ["{{variables.DataSource}}"], + "dataStream": { + "id": "{{dataStreams.dataSourceScans}}", + "name": "dataSourceScans", + "pluginConfigId": "{{configId}}" + }, + "scope": { + "scope": "{{scopes.DataSources}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.DataSource}}" + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": false, + "columnOrder": ["name", "lastRunResult.status", "lastRunResult.startTime", "lastRunResult.endTime", "properties.scanRulesetName", "kind"] + } + } + } + } + }, + { + "static": false, + "w": 4, + "moved": false, + "h": 4, + "x": 0, + "y": 4, + "i": "18f50b03-c364-481b-a341-67f63b5c844b", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Scan Status", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "variables": ["{{variables.DataSource}}"], + "dataStream": { + "id": "{{dataStreams.dataSourceScans}}", + "name": "dataSourceScans", + "pluginConfigId": "{{configId}}" + }, + "scope": { + "scope": "{{scopes.DataSources}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.DataSource}}" + }, + "visualisation": { + "type": "data-stream-blocks", + "config": { + "data-stream-blocks": { + "labelColumn": "name", + "stateColumn": "lastRunResult.status", + "linkColumn": "none", + "columns": 4 + } + } + } + } + } + ] + } +} diff --git a/plugins/MicrosoftPurview/v1/defaultContent/manifest.json b/plugins/MicrosoftPurview/v1/defaultContent/manifest.json new file mode 100644 index 00000000..6fdd2350 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/defaultContent/manifest.json @@ -0,0 +1,7 @@ +{ + "items": [ + { "name": "overview", "type": "dashboard" }, + { "name": "collectionDashboard", "type": "dashboard" }, + { "name": "dataSourceDashboard", "type": "dashboard" } + ] +} diff --git a/plugins/MicrosoftPurview/v1/defaultContent/overview.dash.json b/plugins/MicrosoftPurview/v1/defaultContent/overview.dash.json new file mode 100644 index 00000000..b9dc4f10 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/defaultContent/overview.dash.json @@ -0,0 +1,267 @@ +{ + "name": "Overview", + "schemaVersion": "1.5", + "dashboard": { + "_type": "layout/grid", + "columns": 4, + "version": 1, + "contents": [ + { + "static": false, + "w": 1, + "moved": false, + "h": 2, + "x": 0, + "y": 0, + "i": "3e1f06c3-5cda-4be5-b3e2-0eac7bf464e1", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Total Assets", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "dataStream": { + "id": "{{dataStreams.accountAssetCount}}", + "name": "accountAssetCount", + "pluginConfigId": "{{configId}}" + }, + "visualisation": { + "type": "data-stream-scalar", + "config": { + "data-stream-scalar": { + "value": "totalAssets", + "comparisonColumn": "none", + "label": "assets" + } + } + } + } + }, + { + "static": false, + "w": 1, + "moved": false, + "h": 2, + "x": 1, + "y": 0, + "i": "51587d73-b87d-4fd8-8c6c-9dc637bd691e", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Collections", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "dataStream": { + "id": "{{dataStreams.collections}}", + "name": "collections", + "pluginConfigId": "{{configId}}" + }, + "visualisation": { + "type": "data-stream-scalar", + "config": { + "data-stream-scalar": { + "value": { "type": "count" }, + "comparisonColumn": "none", + "label": "collections" + } + } + } + } + }, + { + "static": false, + "w": 1, + "moved": false, + "h": 2, + "x": 2, + "y": 0, + "i": "415693db-8ef0-41c0-a948-ea5ae0fef955", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Data Sources", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "dataStream": { + "id": "{{dataStreams.listDataSources}}", + "name": "listDataSources", + "pluginConfigId": "{{configId}}" + }, + "visualisation": { + "type": "data-stream-scalar", + "config": { + "data-stream-scalar": { + "value": { "type": "count" }, + "comparisonColumn": "none", + "label": "data sources" + } + } + } + } + }, + { + "static": false, + "w": 1, + "moved": false, + "h": 2, + "x": 3, + "y": 0, + "i": "0ac7deb9-c0cb-4aec-8031-6c91d95ab3c8", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Metadata Policies", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "dataStream": { + "id": "{{dataStreams.metadataPolicies}}", + "name": "metadataPolicies", + "pluginConfigId": "{{configId}}" + }, + "visualisation": { + "type": "data-stream-scalar", + "config": { + "data-stream-scalar": { + "value": { "type": "count" }, + "comparisonColumn": "none", + "label": "policies" + } + } + } + } + }, + { + "static": false, + "w": 2, + "moved": false, + "h": 4, + "x": 0, + "y": 2, + "i": "cae04063-e8b4-4557-9e25-3ad54cc16bc8", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Asset Count by Entity Type", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "dataStream": { + "id": "{{dataStreams.assetsByEntityTypeGlobal}}", + "name": "assetsByEntityTypeGlobal", + "pluginConfigId": "{{configId}}" + }, + "visualisation": { + "type": "data-stream-bar-chart", + "config": { + "data-stream-bar-chart": { + "xAxisData": "entityType", + "yAxisData": ["count"], + "xAxisGroup": "none", + "xAxisLabel": "", + "yAxisLabel": "", + "showXAxisLabel": false, + "showYAxisLabel": false, + "showLegend": false, + "showGrid": true, + "horizontalLayout": "horizontal", + "displayMode": "actual", + "showValue": true, + "grouping": false, + "range": { "type": "auto" } + } + } + } + } + }, + { + "static": false, + "w": 2, + "moved": false, + "h": 4, + "x": 2, + "y": 2, + "i": "898b191e-9d88-48d3-8964-de7e2f59d25e", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Classification Coverage", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "dataStream": { + "id": "{{dataStreams.classificationCoverage}}", + "name": "classificationCoverage", + "pluginConfigId": "{{configId}}" + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": false, + "columnOrder": ["classification", "assets", "coverage"] + } + } + } + } + }, + { + "static": false, + "w": 4, + "moved": false, + "h": 4, + "x": 0, + "y": 6, + "i": "1d34b3f2-89f5-4df2-a280-713b65d94731", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Data Sources", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "dataStream": { + "id": "{{dataStreams.listDataSources}}", + "name": "listDataSources", + "pluginConfigId": "{{configId}}" + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": false, + "columnOrder": ["name", "kind", "collection", "subscriptionUrl"] + } + } + } + } + }, + { + "static": false, + "w": 4, + "moved": false, + "h": 4, + "x": 0, + "y": 10, + "i": "c87c3519-81e9-48b3-831c-3bf19c8f21f3", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Metadata Policies", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "dataStream": { + "id": "{{dataStreams.metadataPolicies}}", + "name": "metadataPolicies", + "pluginConfigId": "{{configId}}" + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": false, + "columnOrder": ["name", "collection", "version", "decisionRuleCount", "attributeRuleCount"] + } + } + } + } + } + ] + } +} diff --git a/plugins/MicrosoftPurview/v1/defaultContent/scopes.json b/plugins/MicrosoftPurview/v1/defaultContent/scopes.json new file mode 100644 index 00000000..a23303c4 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/defaultContent/scopes.json @@ -0,0 +1,32 @@ +[ + { + "name": "Collections", + "matches": { + "sourceType": { + "type": "oneOf", + "values": ["Purview Collection"] + } + }, + "variable": { + "name": "Collection", + "type": "object", + "default": "none", + "allowMultipleSelection": false + } + }, + { + "name": "DataSources", + "matches": { + "sourceType": { + "type": "oneOf", + "values": ["Purview Data Source"] + } + }, + "variable": { + "name": "DataSource", + "type": "object", + "default": "none", + "allowMultipleSelection": false + } + } +] diff --git a/plugins/MicrosoftPurview/v1/docs/README.md b/plugins/MicrosoftPurview/v1/docs/README.md new file mode 100644 index 00000000..7f27595d --- /dev/null +++ b/plugins/MicrosoftPurview/v1/docs/README.md @@ -0,0 +1,66 @@ +# Microsoft Purview + +This plugin connects SquaredUp to Microsoft Purview and surfaces data governance posture across collections, data sources, scans, classifications, policies, and lineage. + +## What it does + +- Imports Purview **Collections**, **Data Sources**, and **Scans** into the SquaredUp graph so they can be used as objects, drilldowns, and dashboard variables. +- Provides data streams for: + - Assets — searchable per collection, with breakdowns by entity type and classification + - Classification coverage — what proportion of assets carry each classification + - Scan status and run history — per data source, per scan, and global recent runs + - Policy coverage — metadata policies attached to collections + - Lineage — upstream and downstream lineage for a given asset +- Ships with out-of-the-box dashboards: an overall **Overview** plus per-object **Collection**, **Data Source**, and **Scan** perspectives. + +## Prerequisites + +You need a Microsoft Entra ID (Azure AD) **app registration** with a **client secret** and at least the **Data Reader** role on the Purview root collection. For scan endpoints, **Data Source Administrator** is required. + +### 1. Register an application in Microsoft Entra ID + +1. In the Azure portal, go to **Microsoft Entra ID → App registrations → New registration**. +2. Give it a name (e.g. `SquaredUp Purview Reader`) and register it (no redirect URI needed). +3. From the **Overview** page, copy the **Application (client) ID** and **Directory (tenant) ID**. +4. Go to **Certificates & secrets → Client secrets → New client secret**. Copy the **Value** immediately — it's only shown once. + +### 2. Grant the application access to your Purview account + +Purview uses **collection-level role assignments**, not Microsoft Graph API permissions. Grant the app registration access inside Purview itself. + +1. Open the **Microsoft Purview governance portal** for your account. +2. Go to **Data Map → Collections → Root collection → Role assignments**. +3. Add the app registration to: + - **Data Reader** (required — read assets, classifications, lineage, search) + - **Data Source Administrator** (required — list data sources, scans, and scan history) + - **Policy Author** or **Collection Administrator** is **not** needed for read-only use, but the app must be able to **read** metadata policies. The Data Source Administrator role is sufficient for this. + +Role assignments at the root collection are inherited by all child collections. + +## Configuration fields + +| Field | What it is | Where to find it | Required | +| --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | -------- | +| Purview account name | The short name of your Purview account — the bit before `.purview.azure.com`. For `contoso-pv.purview.azure.com`, enter `contoso-pv` | Azure portal → your Purview account → **Overview** (the "Name" field) | Yes | +| Directory (tenant) ID | Your Entra ID tenant ID | Entra ID → **Overview** → Tenant ID | Yes | +| Application (client) ID | The app registration's client ID | App registration → **Overview** → Application (client) ID | Yes | +| Client secret | The secret value generated in step 1 | App registration → **Certificates & secrets** | Yes | + +## What gets indexed + +| Object type | Represents | +| ---------------------- | --------------------------------------------------------------------------------------------------------- | +| **Purview Collection** | An organisational unit in your Purview account. The root collection is named after the account itself. | +| **Purview Data Source** | A registered data source — e.g. an Azure SQL database, ADLS Gen2 account, Snowflake instance, Amazon S3 bucket. | +| **Purview Scan** | A configured scan attached to a data source. | + +Assets (tables, files, columns) are **not** indexed because a single Purview account can hold millions of them. They are queried on-demand by the data streams instead. + +## Known limitations + +- **Asset cardinality** — large estates can return millions of assets. The asset search streams page through results but cap at the API's per-call limit of 1000 rows per page. Use the keyword and entity-type filters to narrow results. +- **Lineage depth** — the lineage stream walks up to a configurable depth (default 3 hops) in each direction; very deep lineage graphs are truncated by the API. +- **Policy coverage** — only **metadata policies** are surfaced. DLP and sensitivity-label policies live in Microsoft Graph and are not in scope for this plugin. +- **Permissions are not Graph permissions** — granting roles in Entra ID alone does nothing. The app registration must be added to Purview collection role assignments. +- **Account name format** — the account name field is the bare account name only. Do **not** include `https://` or `.purview.azure.com`. +- **API throttling** — the classic Purview Data Plane has per-account rate limits. Imports can take several minutes for large estates. diff --git a/plugins/MicrosoftPurview/v1/icon.svg b/plugins/MicrosoftPurview/v1/icon.svg new file mode 100644 index 00000000..b8fcf842 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/icon.svg @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/plugins/MicrosoftPurview/v1/indexDefinitions/default.json b/plugins/MicrosoftPurview/v1/indexDefinitions/default.json new file mode 100644 index 00000000..5927c073 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/indexDefinitions/default.json @@ -0,0 +1,36 @@ +{ + "steps": [ + { + "name": "collections", + "dataStream": { "name": "collections" }, + "timeframe": "none", + "objectMapping": { + "id": "name", + "name": "displayName", + "type": "sourceType", + "properties": [ + "collectionName", + { "parentCollection": "parentCollectionName" }, + "description" + ] + } + }, + { + "name": "dataSources", + "dataStream": { "name": "listDataSources" }, + "timeframe": "none", + "objectMapping": { + "id": "name", + "name": "name", + "type": "sourceType", + "properties": [ + "kind", + "collection", + "endpoint", + "resourceGroup", + "subscriptionId" + ] + } + } + ] +} diff --git a/plugins/MicrosoftPurview/v1/metadata.json b/plugins/MicrosoftPurview/v1/metadata.json new file mode 100644 index 00000000..d5fdfa25 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/metadata.json @@ -0,0 +1,70 @@ +{ + "name": "microsoft-purview", + "displayName": "Microsoft Purview", + "version": "1.4.3", + "author": { + "name": "John Hayes", + "type": "community" + }, + "description": "Visualize Microsoft Purview data governance: assets by collection, classification coverage, scan status and history, policy coverage, and lineage.", + "category": "Security", + "type": "hybrid", + "schemaVersion": "2.0", + "keywords": [ + "purview", + "microsoft", + "azure", + "data governance", + "data catalog", + "lineage", + "classification", + "scan", + "data map" + ], + "objectTypes": [ + "Purview Collection", + "Purview Data Source" + ], + "links": [ + { + "category": "documentation", + "url": "https://github.com/squaredup/plugins/blob/main/plugins/MicrosoftPurview/v1/docs/README.md", + "label": "Help adding this plugin" + }, + { + "category": "source", + "url": "https://github.com/squaredup/plugins/tree/main/plugins/MicrosoftPurview/v1", + "label": "Repository" + } + ], + "base": { + "plugin": "WebAPI", + "majorVersion": "1", + "config": { + "baseUrl": "https://{{accountName}}.purview.azure.com/", + "authMode": "oauth2", + "oauth2GrantType": "clientCredentials", + "oauth2TokenUrl": "https://login.microsoftonline.com/{{tenantId}}/oauth2/v2.0/token", + "oauth2ClientId": "{{clientId}}", + "oauth2ClientSecret": "{{clientSecret}}", + "oauth2ClientSecretLocationDuringAuth": "body", + "oauth2Scope": "https://purview.azure.net/.default", + "oauth2TokenExtraHeaders": [ + { + "key": "Content-Type", + "value": "application/x-www-form-urlencoded" + } + ], + "oauth2TokenExtraArgs": [ + { + "key": "grant_type", + "value": "client_credentials" + }, + { + "key": "scope", + "value": "https://purview.azure.net/.default" + } + ] + } + } +} diff --git a/plugins/MicrosoftPurview/v1/ui.json b/plugins/MicrosoftPurview/v1/ui.json new file mode 100644 index 00000000..7c205444 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/ui.json @@ -0,0 +1,39 @@ +[ + { + "type": "text", + "name": "accountName", + "label": "Purview account name", + "placeholder": "e.g. contoso-pv", + "help": "The short name of your Purview account — the part before `.purview.azure.com`. Do **not** include `https://` or the full domain.", + "validation": { + "required": true + } + }, + { + "type": "text", + "name": "tenantId", + "label": "Directory (tenant) ID", + "placeholder": "Enter a directory (tenant) ID", + "validation": { + "required": true + } + }, + { + "type": "text", + "name": "clientId", + "label": "Application (client) ID", + "placeholder": "Enter an application (client) ID", + "validation": { + "required": true + } + }, + { + "type": "password", + "name": "clientSecret", + "label": "Client secret", + "placeholder": "Enter a client secret", + "validation": { + "required": true + } + } +] From 02b33887e656530a5f4ceb055e287021f0978383 Mon Sep 17 00:00:00 2001 From: John Hayes Date: Thu, 23 Jul 2026 14:38:18 +0100 Subject: [PATCH 2/4] Update Microsoft Purview plugin to v1.5.0 - Fix classification summary (count from assets instead of missing facet count field) - Add Classified Assets stream (all classifications, one row per asset-classification pair) - Add Assets with Custom Classifications stream (non-Microsoft classifications only) - Add Custom Classifications stream (account-wide, available on Collection and Data Source) - Update Scans on Data Source stream with dataSourceName field and postRequestScript - Add Duration (s) numeric column to Scan History for average queries - Restrict Scan History to single object selection - Remove Collection and Data Source out-of-the-box dashboards Co-Authored-By: Claude Sonnet 4.6 --- .../collectionClassificationSummary.json | 16 +- .../collectionClassifiedAssets.json | 35 ++++ .../collectionCustomClassifiedAssets.json | 35 ++++ .../v1/dataStreams/customClassifications.json | 25 +++ .../v1/dataStreams/dataSourceScans.json | 31 +-- .../v1/dataStreams/scanHistory.json | 9 + .../v1/dataStreams/scansForDataSource.json | 6 +- .../collectionClassificationSummary.js | 4 +- .../scripts/collectionClassifiedAssets.js | 17 ++ .../collectionCustomClassifiedAssets.js | 17 ++ .../scripts/customClassifications.js | 12 ++ .../v1/dataStreams/scripts/dataSourceScans.js | 6 - .../collectionDashboard.dash.json | 196 ------------------ .../dataSourceDashboard.dash.json | 115 ---------- .../v1/defaultContent/manifest.json | 4 +- plugins/MicrosoftPurview/v1/metadata.json | 2 +- 16 files changed, 165 insertions(+), 365 deletions(-) create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/collectionClassifiedAssets.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/collectionCustomClassifiedAssets.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/customClassifications.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionClassifiedAssets.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionCustomClassifiedAssets.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/customClassifications.js delete mode 100644 plugins/MicrosoftPurview/v1/defaultContent/collectionDashboard.dash.json delete mode 100644 plugins/MicrosoftPurview/v1/defaultContent/dataSourceDashboard.dash.json diff --git a/plugins/MicrosoftPurview/v1/dataStreams/collectionClassificationSummary.json b/plugins/MicrosoftPurview/v1/dataStreams/collectionClassificationSummary.json index 92d91c02..153814c4 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/collectionClassificationSummary.json +++ b/plugins/MicrosoftPurview/v1/dataStreams/collectionClassificationSummary.json @@ -12,17 +12,15 @@ ], "postBody": { "keywords": null, - "limit": 1, + "limit": 1000, "filter": { "collectionId": "{{object.collectionName}}" - }, - "facets": [ - { - "facet": "classification", - "count": 200, - "sort": { "count": "desc" } - } - ] + } + }, + "paging": { + "mode": "token", + "in": { "realm": "payload", "path": "continuationToken" }, + "out": { "realm": "body", "path": "continuationToken" } }, "postRequestScript": "collectionClassificationSummary.js" }, diff --git a/plugins/MicrosoftPurview/v1/dataStreams/collectionClassifiedAssets.json b/plugins/MicrosoftPurview/v1/dataStreams/collectionClassifiedAssets.json new file mode 100644 index 00000000..0490d8fd --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/collectionClassifiedAssets.json @@ -0,0 +1,35 @@ +{ + "name": "collectionClassifiedAssets", + "displayName": "Classified Assets", + "description": "Assets with classifications in the selected collection, one row per asset-classification pair", + "tags": ["Classifications"], + "baseDataSourceName": "httpRequestScopedSingle", + "config": { + "httpMethod": "post", + "endpointPath": "datamap/api/search/query", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "postBody": { + "keywords": null, + "limit": 1000, + "filter": { + "collectionId": "{{object.collectionName}}" + } + }, + "paging": { + "mode": "token", + "in": { "realm": "payload", "path": "continuationToken" }, + "out": { "realm": "body", "path": "continuationToken" } + }, + "postRequestScript": "collectionClassifiedAssets.js" + }, + "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Collection"] } }, + "metadata": [ + { "name": "assetName", "displayName": "Asset", "role": "label" }, + { "name": "entityType", "displayName": "Entity Type" }, + { "name": "collectionId", "displayName": "Collection", "visible": false }, + { "name": "classification", "displayName": "Classification" } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/collectionCustomClassifiedAssets.json b/plugins/MicrosoftPurview/v1/dataStreams/collectionCustomClassifiedAssets.json new file mode 100644 index 00000000..1570fa13 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/collectionCustomClassifiedAssets.json @@ -0,0 +1,35 @@ +{ + "name": "collectionCustomClassifiedAssets", + "displayName": "Assets with Custom Classifications", + "description": "Assets using custom (non-Microsoft) classifications in the selected collection, one row per asset-classification pair", + "tags": ["Classifications"], + "baseDataSourceName": "httpRequestScopedSingle", + "config": { + "httpMethod": "post", + "endpointPath": "datamap/api/search/query", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "postBody": { + "keywords": null, + "limit": 1000, + "filter": { + "collectionId": "{{object.collectionName}}" + } + }, + "paging": { + "mode": "token", + "in": { "realm": "payload", "path": "continuationToken" }, + "out": { "realm": "body", "path": "continuationToken" } + }, + "postRequestScript": "collectionCustomClassifiedAssets.js" + }, + "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Collection"] } }, + "metadata": [ + { "name": "assetName", "displayName": "Asset", "role": "label" }, + { "name": "entityType", "displayName": "Entity Type" }, + { "name": "collectionId", "displayName": "Collection", "visible": false }, + { "name": "classification", "displayName": "Classification" } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/customClassifications.json b/plugins/MicrosoftPurview/v1/dataStreams/customClassifications.json new file mode 100644 index 00000000..d636e650 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/customClassifications.json @@ -0,0 +1,25 @@ +{ + "name": "customClassifications", + "displayName": "Custom Classifications", + "description": "Custom classification definitions for this Purview account", + "tags": ["Classifications"], + "baseDataSourceName": "httpRequestScopedSingle", + "config": { + "httpMethod": "get", + "endpointPath": "datamap/api/atlas/v2/types/typedefs", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "postRequestScript": "customClassifications.js" + }, + "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Collection", "Purview Data Source"] } }, + "metadata": [ + { "name": "name", "displayName": "Classification", "role": "label" }, + { "name": "description", "displayName": "Description" }, + { "name": "createdBy", "displayName": "Created By" }, + { "name": "createTime", "displayName": "Created", "shape": "date" }, + { "name": "updatedBy", "displayName": "Updated By" }, + { "name": "updateTime", "displayName": "Updated", "shape": "date" } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/dataSourceScans.json b/plugins/MicrosoftPurview/v1/dataStreams/dataSourceScans.json index c0d0a906..edd4ed39 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/dataSourceScans.json +++ b/plugins/MicrosoftPurview/v1/dataStreams/dataSourceScans.json @@ -1,7 +1,7 @@ { "name": "dataSourceScans", "displayName": "Scans on Data Source", - "description": "Scans configured on the selected data source with latest run status", + "description": "Scan definitions configured on the selected data source", "tags": ["Data Sources"], "baseDataSourceName": "httpRequestScopedSingle", "config": { @@ -40,35 +40,6 @@ "name": "properties.scanRulesetType", "displayName": "Ruleset Type" }, - { - "name": "lastRunResult.status", - "displayName": "Last Status", - "shape": [ - "state", - { - "map": { - "success": ["Succeeded"], - "error": ["Failed", "TransientFailure"], - "warning": ["Canceled"], - "unknown": ["Accepted", "InProgress"] - } - } - ] - }, - { - "name": "lastRunResult.startTime", - "displayName": "Last Run Started", - "shape": "date" - }, - { - "name": "lastRunResult.endTime", - "displayName": "Last Run Ended", - "shape": "date" - }, - { - "name": "lastRunResult.scanLevelType", - "displayName": "Scan Level" - }, { "name": "creationType", "displayName": "Creation Type" } ], "timeframes": false diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scanHistory.json b/plugins/MicrosoftPurview/v1/dataStreams/scanHistory.json index d5d200e0..8faa2c22 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/scanHistory.json +++ b/plugins/MicrosoftPurview/v1/dataStreams/scanHistory.json @@ -33,9 +33,11 @@ "in": { "realm": "payload", "path": "nextLink" } } }, + "objectLimit": 1, "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Data Source"] } }, "metadata": [ { "name": "id", "displayName": "Run ID", "shape": "guid", "visible": false }, + { "name": "parentId", "displayName": "Scan Definition ID", "shape": "guid", "visible": false }, { "name": "startTime", "displayName": "Started", "shape": "date", "role": "timestamp" }, { "name": "endTime", "displayName": "Ended", "shape": "date" }, { @@ -62,6 +64,13 @@ "valueExpression": "{{ $['startTime'] && $['endTime'] ? (new Date($['endTime']).getTime() - new Date($['startTime']).getTime()) / 1000 : null }}", "shape": ["seconds", { "formatDuration": true, "decimalPlaces": 0 }] }, + { + "name": "durationSeconds", + "displayName": "Duration (s)", + "computed": true, + "valueExpression": "{{ $['startTime'] && $['endTime'] ? (new Date($['endTime']).getTime() - new Date($['startTime']).getTime()) / 1000 : null }}", + "shape": "number" + }, { "name": "discoveryExecutionDetails.statistics.assets.discovered", "displayName": "Assets Discovered", diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scansForDataSource.json b/plugins/MicrosoftPurview/v1/dataStreams/scansForDataSource.json index 647c7fd7..c3224eb9 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/scansForDataSource.json +++ b/plugins/MicrosoftPurview/v1/dataStreams/scansForDataSource.json @@ -20,15 +20,15 @@ "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Data Source"] } }, "metadata": [ { "name": "name", "displayName": "Scan Name", "role": "value", "visible": false }, - { "name": "lastRunResult.discoveryExecutionDetails.discoveryStartTime", "displayName": "Last Run Started", "shape": "date", "visible": false }, + { "name": "lastRunResult.startTime", "displayName": "Last Run Started", "shape": "date", "visible": false }, { "name": "scanLabel", "displayName": "Scan", "role": "label", "computed": true, - "valueExpression": "{{ $['lastRunResult.discoveryExecutionDetails.discoveryStartTime'] ? $['name'] + ' — ' + new Date($['lastRunResult.discoveryExecutionDetails.discoveryStartTime']).toISOString().slice(0, 10) : $['name'] + ' — never run' }}" + "valueExpression": "{{ $['lastRunResult.startTime'] ? $['name'] + ' — ' + new Date($['lastRunResult.startTime']).toISOString().slice(0, 10) : $['name'] + ' — never run' }}" } ], "timeframes": false, - "defaultShaping": { "sort": { "by": [["lastRunResult.discoveryExecutionDetails.discoveryStartTime", "desc"]] } } + "defaultShaping": { "sort": { "by": [["lastRunResult.startTime", "desc"]] } } } diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionClassificationSummary.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionClassificationSummary.js index 82daace8..447ca8cf 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionClassificationSummary.js +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionClassificationSummary.js @@ -1,6 +1,6 @@ +const assets = data?.value || []; const total = Number(data?.['@search.count']) || 0; -const facets = data?.['@search.facets']?.classification || []; -const classified = facets.reduce((sum, f) => sum + (Number(f.count) || 0), 0); +const classified = assets.filter(a => (a.classification?.length ?? 0) > 0).length; const unclassified = Math.max(0, total - classified); result = [ diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionClassifiedAssets.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionClassifiedAssets.js new file mode 100644 index 00000000..9b5b7dac --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionClassifiedAssets.js @@ -0,0 +1,17 @@ +const assets = data?.value || []; +const rows = []; + +for (const asset of assets) { + const classifications = asset.classification || []; + if (classifications.length === 0) continue; + for (const cls of classifications) { + rows.push({ + assetName: asset.name || '', + entityType: asset.entityType || '', + collectionId: asset.collectionId || '', + classification: cls, + }); + } +} + +result = rows; diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionCustomClassifiedAssets.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionCustomClassifiedAssets.js new file mode 100644 index 00000000..b3c00200 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionCustomClassifiedAssets.js @@ -0,0 +1,17 @@ +const assets = data?.value || []; +const rows = []; + +for (const asset of assets) { + const classifications = (asset.classification || []).filter(c => !c.startsWith('MICROSOFT.')); + if (classifications.length === 0) continue; + for (const cls of classifications) { + rows.push({ + assetName: asset.name || '', + entityType: asset.entityType || '', + collectionId: asset.collectionId || '', + classification: cls, + }); + } +} + +result = rows; diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/customClassifications.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/customClassifications.js new file mode 100644 index 00000000..6ef568a2 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/customClassifications.js @@ -0,0 +1,12 @@ +const defs = data?.classificationDefs || []; + +result = defs + .filter(c => !c.name?.startsWith('MICROSOFT.')) + .map(c => ({ + name: c.name || '', + description: c.description || '', + createdBy: c.createdBy || '', + createTime: c.createTime ? new Date(c.createTime).toISOString() : null, + updatedBy: c.updatedBy || '', + updateTime: c.updateTime ? new Date(c.updateTime).toISOString() : null, + })); diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/dataSourceScans.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/dataSourceScans.js index 97279c43..e2d144dd 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/scripts/dataSourceScans.js +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/dataSourceScans.js @@ -1,8 +1,6 @@ const scans = data?.value || []; result = scans.map((scan) => { - const lr = scan.lastRunResult || {}; - const discovery = lr.discoveryExecutionDetails || {}; const props = scan.properties || {}; return { @@ -13,9 +11,5 @@ result = scans.map((scan) => { 'properties.collection.referenceName': props.collection?.referenceName || '', 'properties.scanRulesetName': props.scanRulesetName || '', 'properties.scanRulesetType': props.scanRulesetType || '', - 'lastRunResult.status': lr.scanRunStatus || '', - 'lastRunResult.startTime': discovery.discoveryStartTime || null, - 'lastRunResult.endTime': discovery.discoveryEndTime || null, - 'lastRunResult.scanLevelType': lr.scanLevelType || '', }; }); diff --git a/plugins/MicrosoftPurview/v1/defaultContent/collectionDashboard.dash.json b/plugins/MicrosoftPurview/v1/defaultContent/collectionDashboard.dash.json deleted file mode 100644 index 7c4c53e3..00000000 --- a/plugins/MicrosoftPurview/v1/defaultContent/collectionDashboard.dash.json +++ /dev/null @@ -1,196 +0,0 @@ -{ - "name": "Collection", - "schemaVersion": "1.5", - "variables": ["{{variables.Collection}}"], - "dashboard": { - "_type": "layout/grid", - "columns": 4, - "version": 1, - "contents": [ - { - "static": false, - "w": 1, - "moved": false, - "h": 4, - "x": 0, - "y": 0, - "i": "25f69cbd-2d27-4175-8273-75f70bea420d", - "z": 0, - "config": { - "_type": "tile/data-stream", - "title": "Properties", - "description": "", - "activePluginConfigIds": ["{{configId}}"], - "variables": ["{{variables.Collection}}"], - "dataStream": { "id": "datastream-properties" }, - "scope": { - "scope": "{{scopes.Collections}}", - "workspace": "{{workspaceId}}", - "variable": "{{variables.Collection}}" - }, - "visualisation": { - "type": "data-stream-table", - "config": { - "data-stream-table": { "transpose": true } - } - } - } - }, - { - "static": false, - "w": 3, - "moved": false, - "h": 4, - "x": 1, - "y": 0, - "i": "5c456e3c-637a-4aea-b0a6-318dbbf4641e", - "z": 0, - "config": { - "_type": "tile/data-stream", - "title": "Classification Summary", - "description": "", - "activePluginConfigIds": ["{{configId}}"], - "variables": ["{{variables.Collection}}"], - "dataStream": { - "id": "{{dataStreams.collectionClassificationSummary}}", - "name": "collectionClassificationSummary", - "pluginConfigId": "{{configId}}" - }, - "scope": { - "scope": "{{scopes.Collections}}", - "workspace": "{{workspaceId}}", - "variable": "{{variables.Collection}}" - }, - "visualisation": { - "type": "data-stream-donut-chart", - "config": { - "data-stream-donut-chart": { - "valueColumn": "assets", - "labelColumn": "label", - "showValuesAsPercentage": true, - "legendPosition": "auto", - "legendMode": "table" - } - } - } - } - }, - { - "static": false, - "w": 2, - "moved": false, - "h": 5, - "x": 0, - "y": 4, - "i": "3b4de95e-0982-4058-8965-d362452e4150", - "z": 0, - "config": { - "_type": "tile/data-stream", - "title": "Asset Count by Entity Type", - "description": "", - "activePluginConfigIds": ["{{configId}}"], - "variables": ["{{variables.Collection}}"], - "dataStream": { - "id": "{{dataStreams.assetCountByEntityType}}", - "name": "assetCountByEntityType", - "pluginConfigId": "{{configId}}" - }, - "scope": { - "scope": "{{scopes.Collections}}", - "workspace": "{{workspaceId}}", - "variable": "{{variables.Collection}}" - }, - "visualisation": { - "type": "data-stream-bar-chart", - "config": { - "data-stream-bar-chart": { - "xAxisData": "entityType", - "yAxisData": ["count"], - "xAxisGroup": "none", - "showLegend": false, - "showGrid": true, - "horizontalLayout": "horizontal", - "displayMode": "actual", - "showValue": true, - "grouping": false, - "range": { "type": "auto" } - } - } - } - } - }, - { - "static": false, - "w": 2, - "moved": false, - "h": 5, - "x": 2, - "y": 4, - "i": "f335dba3-f381-4154-89cd-46674611bc64", - "z": 0, - "config": { - "_type": "tile/data-stream", - "title": "Policy Coverage", - "description": "", - "activePluginConfigIds": ["{{configId}}"], - "variables": ["{{variables.Collection}}"], - "dataStream": { - "id": "{{dataStreams.collectionPolicyCoverage}}", - "name": "collectionPolicyCoverage", - "pluginConfigId": "{{configId}}" - }, - "scope": { - "scope": "{{scopes.Collections}}", - "workspace": "{{workspaceId}}", - "variable": "{{variables.Collection}}" - }, - "visualisation": { - "type": "data-stream-table", - "config": { - "data-stream-table": { - "transpose": false, - "columnOrder": ["state", "name", "version", "decisionRuleCount", "attributeRuleCount"] - } - } - } - } - }, - { - "static": false, - "w": 4, - "moved": false, - "h": 5, - "x": 0, - "y": 9, - "i": "1f931a39-24c6-416c-b564-75bb490bcf5c", - "z": 0, - "config": { - "_type": "tile/data-stream", - "title": "Assets in Collection", - "description": "", - "activePluginConfigIds": ["{{configId}}"], - "variables": ["{{variables.Collection}}"], - "dataStream": { - "id": "{{dataStreams.assetsByCollection}}", - "name": "assetsByCollection", - "pluginConfigId": "{{configId}}" - }, - "scope": { - "scope": "{{scopes.Collections}}", - "workspace": "{{workspaceId}}", - "variable": "{{variables.Collection}}" - }, - "visualisation": { - "type": "data-stream-table", - "config": { - "data-stream-table": { - "transpose": false, - "columnOrder": ["name", "entityType", "assetType", "classification", "term", "qualifiedName"] - } - } - } - } - } - ] - } -} diff --git a/plugins/MicrosoftPurview/v1/defaultContent/dataSourceDashboard.dash.json b/plugins/MicrosoftPurview/v1/defaultContent/dataSourceDashboard.dash.json deleted file mode 100644 index 5f944db3..00000000 --- a/plugins/MicrosoftPurview/v1/defaultContent/dataSourceDashboard.dash.json +++ /dev/null @@ -1,115 +0,0 @@ -{ - "name": "Data Source", - "schemaVersion": "1.5", - "variables": ["{{variables.DataSource}}"], - "dashboard": { - "_type": "layout/grid", - "columns": 4, - "version": 1, - "contents": [ - { - "static": false, - "w": 1, - "moved": false, - "h": 4, - "x": 0, - "y": 0, - "i": "0f4d8de5-c60c-4fa7-beb7-76260f18e504", - "z": 0, - "config": { - "_type": "tile/data-stream", - "title": "Properties", - "description": "", - "activePluginConfigIds": ["{{configId}}"], - "variables": ["{{variables.DataSource}}"], - "dataStream": { "id": "datastream-properties" }, - "scope": { - "scope": "{{scopes.DataSources}}", - "workspace": "{{workspaceId}}", - "variable": "{{variables.DataSource}}" - }, - "visualisation": { - "type": "data-stream-table", - "config": { - "data-stream-table": { "transpose": true } - } - } - } - }, - { - "static": false, - "w": 3, - "moved": false, - "h": 4, - "x": 1, - "y": 0, - "i": "07d8ec4e-404c-4925-94c1-01a1da2e9938", - "z": 0, - "config": { - "_type": "tile/data-stream", - "title": "Scans", - "description": "", - "activePluginConfigIds": ["{{configId}}"], - "variables": ["{{variables.DataSource}}"], - "dataStream": { - "id": "{{dataStreams.dataSourceScans}}", - "name": "dataSourceScans", - "pluginConfigId": "{{configId}}" - }, - "scope": { - "scope": "{{scopes.DataSources}}", - "workspace": "{{workspaceId}}", - "variable": "{{variables.DataSource}}" - }, - "visualisation": { - "type": "data-stream-table", - "config": { - "data-stream-table": { - "transpose": false, - "columnOrder": ["name", "lastRunResult.status", "lastRunResult.startTime", "lastRunResult.endTime", "properties.scanRulesetName", "kind"] - } - } - } - } - }, - { - "static": false, - "w": 4, - "moved": false, - "h": 4, - "x": 0, - "y": 4, - "i": "18f50b03-c364-481b-a341-67f63b5c844b", - "z": 0, - "config": { - "_type": "tile/data-stream", - "title": "Scan Status", - "description": "", - "activePluginConfigIds": ["{{configId}}"], - "variables": ["{{variables.DataSource}}"], - "dataStream": { - "id": "{{dataStreams.dataSourceScans}}", - "name": "dataSourceScans", - "pluginConfigId": "{{configId}}" - }, - "scope": { - "scope": "{{scopes.DataSources}}", - "workspace": "{{workspaceId}}", - "variable": "{{variables.DataSource}}" - }, - "visualisation": { - "type": "data-stream-blocks", - "config": { - "data-stream-blocks": { - "labelColumn": "name", - "stateColumn": "lastRunResult.status", - "linkColumn": "none", - "columns": 4 - } - } - } - } - } - ] - } -} diff --git a/plugins/MicrosoftPurview/v1/defaultContent/manifest.json b/plugins/MicrosoftPurview/v1/defaultContent/manifest.json index 6fdd2350..1c0385b4 100644 --- a/plugins/MicrosoftPurview/v1/defaultContent/manifest.json +++ b/plugins/MicrosoftPurview/v1/defaultContent/manifest.json @@ -1,7 +1,5 @@ { "items": [ - { "name": "overview", "type": "dashboard" }, - { "name": "collectionDashboard", "type": "dashboard" }, - { "name": "dataSourceDashboard", "type": "dashboard" } + { "name": "overview", "type": "dashboard" } ] } diff --git a/plugins/MicrosoftPurview/v1/metadata.json b/plugins/MicrosoftPurview/v1/metadata.json index d5fdfa25..94ca868c 100644 --- a/plugins/MicrosoftPurview/v1/metadata.json +++ b/plugins/MicrosoftPurview/v1/metadata.json @@ -1,7 +1,7 @@ { "name": "microsoft-purview", "displayName": "Microsoft Purview", - "version": "1.4.3", + "version": "1.5.0", "author": { "name": "John Hayes", "type": "community" From 2992e3cbb4e6992fe0f42e372c37dc6106717289 Mon Sep 17 00:00:00 2001 From: John Hayes Date: Thu, 23 Jul 2026 16:53:22 +0100 Subject: [PATCH 3/4] Update Overview dashboard layout to v1.5.1 Resize Classification Coverage and Metadata Policies tiles to 2x2 and stack them on the right side of the Asset Count by Entity Type tile. Co-Authored-By: Claude Sonnet 4.6 --- .../v1/defaultContent/overview.dash.json | 10 +++++----- plugins/MicrosoftPurview/v1/metadata.json | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/MicrosoftPurview/v1/defaultContent/overview.dash.json b/plugins/MicrosoftPurview/v1/defaultContent/overview.dash.json index b9dc4f10..a0e34139 100644 --- a/plugins/MicrosoftPurview/v1/defaultContent/overview.dash.json +++ b/plugins/MicrosoftPurview/v1/defaultContent/overview.dash.json @@ -176,7 +176,7 @@ "static": false, "w": 2, "moved": false, - "h": 4, + "h": 2, "x": 2, "y": 2, "i": "898b191e-9d88-48d3-8964-de7e2f59d25e", @@ -234,11 +234,11 @@ }, { "static": false, - "w": 4, + "w": 2, "moved": false, - "h": 4, - "x": 0, - "y": 10, + "h": 2, + "x": 2, + "y": 4, "i": "c87c3519-81e9-48b3-831c-3bf19c8f21f3", "z": 0, "config": { diff --git a/plugins/MicrosoftPurview/v1/metadata.json b/plugins/MicrosoftPurview/v1/metadata.json index 94ca868c..af4267de 100644 --- a/plugins/MicrosoftPurview/v1/metadata.json +++ b/plugins/MicrosoftPurview/v1/metadata.json @@ -1,7 +1,7 @@ { "name": "microsoft-purview", "displayName": "Microsoft Purview", - "version": "1.5.0", + "version": "1.5.1", "author": { "name": "John Hayes", "type": "community" From 3952610b2059272736df10cdda1754287f0e121d Mon Sep 17 00:00:00 2001 From: John Hayes Date: Thu, 23 Jul 2026 18:10:05 +0100 Subject: [PATCH 4/4] Add Governance Domains and Glossary Terms streams to v1.6.2 - Add Governance Domains stream (Unified Catalog API, account-wide) - Add Glossary Terms stream scoped to Purview Governance Domain objects - Index Governance Domains as a new object type - Strip HTML from description fields in both streams - Update README with Unified Catalog permission requirements - Add Purview Governance Domain to objectTypes and What gets indexed Co-Authored-By: Claude Sonnet 4.6 --- .../v1/dataStreams/glossaryTerms.json | 33 +++++++++++++++++++ .../v1/dataStreams/governanceDomains.json | 33 +++++++++++++++++++ .../v1/dataStreams/scripts/glossaryTerms.js | 14 ++++++++ .../dataStreams/scripts/governanceDomains.js | 16 +++++++++ plugins/MicrosoftPurview/v1/docs/README.md | 17 +++++++++- .../v1/indexDefinitions/default.json | 15 +++++++++ plugins/MicrosoftPurview/v1/metadata.json | 5 +-- 7 files changed, 130 insertions(+), 3 deletions(-) create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/glossaryTerms.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/governanceDomains.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/glossaryTerms.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/governanceDomains.js diff --git a/plugins/MicrosoftPurview/v1/dataStreams/glossaryTerms.json b/plugins/MicrosoftPurview/v1/dataStreams/glossaryTerms.json new file mode 100644 index 00000000..7808fd79 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/glossaryTerms.json @@ -0,0 +1,33 @@ +{ + "name": "glossaryTerms", + "displayName": "Glossary Terms", + "description": "Glossary terms defined in the selected governance domain", + "tags": ["Governance"], + "baseDataSourceName": "httpRequestScopedSingle", + "objectLimit": 1, + "config": { + "httpMethod": "get", + "endpointPath": "datagovernance/catalog/terms", + "getArgs": [ + { "key": "api-version", "value": "2026-03-20-preview" }, + { "key": "domainId", "value": "{{object.rawId}}" } + ], + "postRequestScript": "glossaryTerms.js", + "paging": { + "mode": "nextUrl", + "in": { "realm": "payload", "path": "nextLink" } + } + }, + "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Governance Domain"] } }, + "metadata": [ + { "name": "id", "displayName": "ID", "shape": "guid", "visible": false }, + { "name": "name", "displayName": "Term", "role": "label" }, + { "name": "description", "displayName": "Description" }, + { "name": "status", "displayName": "Status" }, + { "name": "acronyms", "displayName": "Acronyms" }, + { "name": "parentId", "displayName": "Parent Term ID", "shape": "guid", "visible": false }, + { "name": "createdAt", "displayName": "Created", "shape": "date" }, + { "name": "lastModifiedAt", "displayName": "Last Modified", "shape": "date" } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/governanceDomains.json b/plugins/MicrosoftPurview/v1/dataStreams/governanceDomains.json new file mode 100644 index 00000000..c3d85d73 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/governanceDomains.json @@ -0,0 +1,33 @@ +{ + "name": "governanceDomains", + "displayName": "Governance Domains", + "description": "Governance domains defined in the Purview Unified Catalog", + "tags": ["Governance"], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "get", + "endpointPath": "datagovernance/catalog/businessdomains", + "getArgs": [ + { "key": "api-version", "value": "2026-03-20-preview" } + ], + "postRequestScript": "governanceDomains.js", + "paging": { + "mode": "nextUrl", + "in": { "realm": "payload", "path": "nextLink" } + } + }, + "matches": "none", + "metadata": [ + { "name": "id", "displayName": "ID", "shape": "guid", "visible": false }, + { "name": "name", "displayName": "Domain", "role": "label" }, + { "name": "description", "displayName": "Description" }, + { "name": "type", "displayName": "Type" }, + { "name": "status", "displayName": "Status" }, + { "name": "parentId", "displayName": "Parent ID", "shape": "guid", "visible": false }, + { "name": "isRestricted", "displayName": "Restricted" }, + { "name": "createdAt", "displayName": "Created", "shape": "date" }, + { "name": "lastModifiedAt", "displayName": "Last Modified", "shape": "date" }, + { "name": "sourceType", "displayName": "Source Type", "visible": false } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/glossaryTerms.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/glossaryTerms.js new file mode 100644 index 00000000..30f64fba --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/glossaryTerms.js @@ -0,0 +1,14 @@ +const stripHtml = s => (s || '').replace(/<[^>]*>/g, '').replace(/ /g, ' ').trim() || null; + +const terms = data?.value || []; + +result = terms.map(term => ({ + id: term.id || '', + name: term.name || '', + description: stripHtml(term.description), + status: term.status || '', + acronyms: (term.acronyms || []).join(', ') || null, + parentId: term.parentId || null, + createdAt: term.systemData?.createdAt || null, + lastModifiedAt: term.systemData?.lastModifiedAt || null, +})); diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/governanceDomains.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/governanceDomains.js new file mode 100644 index 00000000..95736ab8 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/governanceDomains.js @@ -0,0 +1,16 @@ +const stripHtml = s => (s || '').replace(/<[^>]*>/g, '').replace(/ /g, ' ').trim() || null; + +const domains = data?.value || []; + +result = domains.map(d => ({ + id: d.id || '', + name: d.name || '', + description: stripHtml(d.description), + type: d.type || '', + status: d.status || '', + parentId: d.parentId || null, + isRestricted: d.isRestricted ?? null, + createdAt: d.systemData?.createdAt || null, + lastModifiedAt: d.systemData?.lastModifiedAt || null, + sourceType: 'Purview Governance Domain', +})); diff --git a/plugins/MicrosoftPurview/v1/docs/README.md b/plugins/MicrosoftPurview/v1/docs/README.md index 7f27595d..af6f6dc2 100644 --- a/plugins/MicrosoftPurview/v1/docs/README.md +++ b/plugins/MicrosoftPurview/v1/docs/README.md @@ -8,10 +8,13 @@ This plugin connects SquaredUp to Microsoft Purview and surfaces data governance - Provides data streams for: - Assets — searchable per collection, with breakdowns by entity type and classification - Classification coverage — what proportion of assets carry each classification + - Classifications — custom and built-in classifications applied to assets - Scan status and run history — per data source, per scan, and global recent runs - Policy coverage — metadata policies attached to collections - Lineage — upstream and downstream lineage for a given asset -- Ships with out-of-the-box dashboards: an overall **Overview** plus per-object **Collection**, **Data Source**, and **Scan** perspectives. + - Governance domains — domains defined in the Unified Catalog + - Glossary terms — terms defined within a governance domain +- Ships with an out-of-the-box **Overview** dashboard. ## Prerequisites @@ -37,6 +40,17 @@ Purview uses **collection-level role assignments**, not Microsoft Graph API perm Role assignments at the root collection are inherited by all child collections. +### 3. Grant access to the Unified Catalog (governance domains and glossary terms) + +The **Governance Domains** and **Glossary Terms** streams use the Purview Unified Catalog API, which has its own permission model separate from Data Map collection roles. + +1. Open the **Microsoft Purview portal** and go to **Unified Catalog → Catalog Management → Governance Domains**. +2. Select the governance domain you want to expose. +3. Go to the **Roles** tab. +4. Add the app registration to the **Data Catalog Reader** role. + +Repeat for each governance domain you want to surface. Without this role, the Governance Domains and Glossary Terms streams will return an "Unauthorized" error even if all Data Map roles are correctly assigned. + ## Configuration fields | Field | What it is | Where to find it | Required | @@ -53,6 +67,7 @@ Role assignments at the root collection are inherited by all child collections. | **Purview Collection** | An organisational unit in your Purview account. The root collection is named after the account itself. | | **Purview Data Source** | A registered data source — e.g. an Azure SQL database, ADLS Gen2 account, Snowflake instance, Amazon S3 bucket. | | **Purview Scan** | A configured scan attached to a data source. | +| **Purview Governance Domain** | A governance domain defined in the Unified Catalog. Requires the Data Catalog Reader role — see step 3 above. | Assets (tables, files, columns) are **not** indexed because a single Purview account can hold millions of them. They are queried on-demand by the data streams instead. diff --git a/plugins/MicrosoftPurview/v1/indexDefinitions/default.json b/plugins/MicrosoftPurview/v1/indexDefinitions/default.json index 5927c073..ec30fb99 100644 --- a/plugins/MicrosoftPurview/v1/indexDefinitions/default.json +++ b/plugins/MicrosoftPurview/v1/indexDefinitions/default.json @@ -31,6 +31,21 @@ "subscriptionId" ] } + }, + { + "name": "governanceDomains", + "dataStream": { "name": "governanceDomains" }, + "timeframe": "none", + "objectMapping": { + "id": "id", + "name": "name", + "type": "sourceType", + "properties": [ + "description", + "type", + "status" + ] + } } ] } diff --git a/plugins/MicrosoftPurview/v1/metadata.json b/plugins/MicrosoftPurview/v1/metadata.json index af4267de..ee06798b 100644 --- a/plugins/MicrosoftPurview/v1/metadata.json +++ b/plugins/MicrosoftPurview/v1/metadata.json @@ -1,7 +1,7 @@ { "name": "microsoft-purview", "displayName": "Microsoft Purview", - "version": "1.5.1", + "version": "1.6.2", "author": { "name": "John Hayes", "type": "community" @@ -23,7 +23,8 @@ ], "objectTypes": [ "Purview Collection", - "Purview Data Source" + "Purview Data Source", + "Purview Governance Domain" ], "links": [ {