Azure cloud resource crawling: Multiple resources are not being merged into single snapshot file based on masterSnapshotId array at master-compliance-test.json

[rezoan]

example: PR-AZR-CLD-KV-009, PR-AZR-CLD-FRD-001, PR-AZR-CLD-AFW-001

[jaiminswan]

Yet to start

[rezoan]

@vatsalgit5118 seems the issue still exist but in a differet way:

For example, PR-AZR-CLD-KV-009 we have said it has masterSnapshotId: "AZRSNP_228", "AZRSNP_500" in compliance-test.json which is a merge request of both into a single file.

original resource template: https://portal.azure.com/#@prancerenterprise.com/resource/subscriptions/a6941677-4c37-42fb-960c-dad8f25060a3/resourceGroups/farshid-test/providers/Microsoft.KeyVault/vaults/prancerkv0001/overview

when running the test, i did saw it generates snapshot of AZRSNP_228 into a seperate file as

{
  "structure": "azure",
  "reference": "Prancer Sandbox",
  "contentType": "json",
  "source": "azureConnector",
  "path": "/subscriptions/a6941677-4c37-42fb-960c-dad8f25060a3/resourceGroups/farshid-test/providers/Microsoft.KeyVault/vaults/prancerkv0001",
  "timestamp": 1672206915928,
  "queryuser": "whitekite-spn-shahin",
  "checksum": "be35cd3b4cac3663580cdfdbb141e80d",
  "node": {
    "masterSnapshotId": [
      "AZRSNP_228"
    ],
    "type": "Microsoft.KeyVault/vaults",
    "collection": "Microsoft.KeyVault",
    "version": "2021-06-01-preview",
    "snapshotId": "AZRSNP_228123",
    "path": "/subscriptions/a6941677-4c37-42fb-960c-dad8f25060a3/resourceGroups/farshid-test/providers/Microsoft.KeyVault/vaults/prancerkv0001",
    "validate": true,
    "status": "active"
  },
  "snapshotId": "AZRSNP_228123",
  "mastersnapshot": false,
  "masterSnapshotId": null,
  "collection": "microsoftkeyvault",
  "region": "eastus2",
  "session_id": "session_1672234362172",
  "json": {
    "resources": [
      {
        "id": "/subscriptions/a6941677-4c37-42fb-960c-dad8f25060a3/resourceGroups/farshid-test/providers/Microsoft.KeyVault/vaults/prancerkv0001",
        "name": "prancerkv0001",
        "type": "Microsoft.KeyVault/vaults",
        "location": "eastus2",
        "tags": {},
        "properties": {
          "sku": {
            "family": "A",
            "name": "Standard"
          },
          "tenantId": "2367bdec-cf51-44b1-a8db-3677de1acc38",
          "privateEndpointConnections": [
            {
              "id": "/subscriptions/a6941677-4c37-42fb-960c-dad8f25060a3/resourceGroups/farshid-test/providers/Microsoft.KeyVault/vaults/prancerkv0001/privateEndpointConnections/test-pe",
              "properties": {
                "provisioningState": "Succeeded",
                "privateEndpoint": {
                  "id": "/subscriptions/a6941677-4c37-42fb-960c-dad8f25060a3/resourceGroups/farshid-test/providers/Microsoft.Network/privateEndpoints/test-pe"
                },
                "privateLinkServiceConnectionState": {
                  "status": "Approved",
                  "actionsRequired": "None"
                }
              }
            }
          ],
          "accessPolicies": [
            {
              "tenantId": "2367bdec-cf51-44b1-a8db-3677de1acc38",
              "objectId": "142c515d-7900-46df-86dd-9f81a541867e",
              "permissions": {
                "keys": [
                  "Get",
                  "List",
                  "Update",
                  "Create",
                  "Import",
                  "Delete",
                  "Recover",
                  "Backup",
                  "Restore",
                  "GetRotationPolicy",
                  "SetRotationPolicy",
                  "Rotate"
                ],
                "secrets": [
                  "Get",
                  "List",
                  "Set",
                  "Delete",
                  "Recover",
                  "Backup",
                  "Restore"
                ],
                "certificates": [
                  "Get",
                  "List",
                  "Update",
                  "Create",
                  "Import",
                  "Delete",
                  "Recover",
                  "Backup",
                  "Restore",
                  "ManageContacts",
                  "ManageIssuers",
                  "GetIssuers",
                  "ListIssuers",
                  "SetIssuers",
                  "DeleteIssuers"
                ]
              }
            },
            {
              "tenantId": "2367bdec-cf51-44b1-a8db-3677de1acc38",
              "objectId": "5dc43473-afe0-47ec-93ab-2244a2396a85",
              "permissions": {
                "keys": [],
                "secrets": [
                  "Get",
                  "List"
                ],
                "certificates": []
              }
            },
            {
              "tenantId": "2367bdec-cf51-44b1-a8db-3677de1acc38",
              "objectId": "2b07db87-1fe6-4ca7-ad29-a5e39d35b19e",
              "permissions": {
                "keys": [
                  "Get",
                  "List",
                  "Update",
                  "Create",
                  "Import",
                  "Delete",
                  "Recover",
                  "Backup",
                  "Restore"
                ],
                "secrets": [
                  "Get",
                  "List",
                  "Set",
                  "Delete",
                  "Recover",
                  "Backup",
                  "Restore"
                ],
                "certificates": [
                  "Get",
                  "List",
                  "Update",
                  "Create",
                  "Import",
                  "Delete",
                  "Recover",
                  "Backup",
                  "Restore",
                  "ManageContacts",
                  "ManageIssuers",
                  "GetIssuers",
                  "ListIssuers",
                  "SetIssuers",
                  "DeleteIssuers"
                ]
              }
            }
          ],
          "enabledForDeployment": false,
          "enabledForDiskEncryption": false,
          "enabledForTemplateDeployment": false,
          "enableSoftDelete": true,
          "softDeleteRetentionInDays": 90,
          "enableRbacAuthorization": false,
          "vaultUri": "https://prancerkv0001.vault.azure.net/",
          "provisioningState": "Succeeded",
          "publicNetworkAccess": "Enabled"
        }
      }
    ],
    "subscription_id": "a6941677-4c37-42fb-960c-dad8f25060a3",
    "resource_group": "farshid-test"
  }
}

It has privateEndpointConnections as property (which is wrong as keyvault does not have such property on its own) instead of separate related resource type (this is the issue.)

if we look at the snapshot of AZRSNP_500 it generates like:

{
  "structure": "azure",
  "reference": "Prancer Sandbox",
  "contentType": "json",
  "source": "azureConnector",
  "path": "/subscriptions/a6941677-4c37-42fb-960c-dad8f25060a3/resourceGroups/farshid-test/providers/Microsoft.Network/privateEndpoints/test-pe",
  "timestamp": 1672207123368,
  "queryuser": "whitekite-spn-shahin",
  "checksum": "99914b932bd37a50b983c5e7c90ae93b",
  "node": {
    "masterSnapshotId": [
      "AZRSNP_500"
    ],
    "type": "Microsoft.Network/privateEndpoints",
    "collection": "Microsoft.Network",
    "version": "2021-05-01",
    "snapshotId": "AZRSNP_500130",
    "path": "/subscriptions/a6941677-4c37-42fb-960c-dad8f25060a3/resourceGroups/farshid-test/providers/Microsoft.Network/privateEndpoints/test-pe",
    "validate": true,
    "status": "active"
  },
  "snapshotId": "AZRSNP_500130",
  "mastersnapshot": false,
  "masterSnapshotId": null,
  "collection": "microsoftnetwork",
  "region": "",
  "session_id": "session_1672234362172",
  "json": {
    "resources": [
      {
        "type": "Microsoft.Network/privateEndpoints",
        "apiVersion": "2022-05-01",
        "name": "test-pe",
        "location": "eastus",
        "properties": {
          "privateLinkServiceConnections": [
            {
              "name": "test-pe",
              "id": "Microsoft.Network/privateEndpoints/test-pe/privateLinkServiceConnections/test-pe",
              "properties": {
                "privateLinkServiceId": "/subscriptions/a6941677-4c37-42fb-960c-dad8f25060a3/resourceGroups/farshid-test/providers/Microsoft.KeyVault/vaults/prancerkv0001",
                "groupIds": [
                  "vault"
                ],
                "privateLinkServiceConnectionState": {
                  "status": "Approved",
                  "actionsRequired": "None"
                }
              }
            }
          ],
          "manualPrivateLinkServiceConnections": [],
          "subnet": {
            "id": "/subscriptions/a6941677-4c37-42fb-960c-dad8f25060a3/resourceGroups/dev-rezoan/providers/Microsoft.Network/virtualNetworks/dev-rezoan-vnet/subnets/default"
          },
          "ipConfigurations": [],
          "customDnsConfigs": []
        }
      },
      {
        "type": "Microsoft.Network/privateEndpoints/privateDnsZoneGroups",
        "apiVersion": "2022-05-01",
        "name": "test-pe/default",
        "dependsOn": [
          "Microsoft.Network/privateEndpoints/test-pe"
        ],
        "properties": {
          "privateDnsZoneConfigs": [
            {
              "name": "privatelink-vaultcore-azure-net",
              "properties": {
                "privateDnsZoneId": "/subscriptions/a6941677-4c37-42fb-960c-dad8f25060a3/resourceGroups/cloud-shell-storage-centralindia/providers/Microsoft.Network/privateDnsZones/privatelink.vaultcore.azure.net"
              }
            }
          ]
        }
      }
    ],
    "subscription_id": "a6941677-4c37-42fb-960c-dad8f25060a3",
    "resource_group": "farshid-test"
  }
}

Idea was to merge both 2 file into a single one without introducing any new property to any resource type which does not exist in original resource template.