diff --git a/Documentation/guides/FastDeploy2.md b/Documentation/guides/FastDeploy2.md index 88626b32381..38feb810acc 100644 --- a/Documentation/guides/FastDeploy2.md +++ b/Documentation/guides/FastDeploy2.md @@ -23,9 +23,8 @@ properties intended for end users are: | `$(_AndroidFastDeployAppFileTransferMode)` | `Symlink` (for `FastDeploy2`) | How staged files are surfaced in the override directory: `Symlink` or `Copy`. | | `$(AndroidFastDeploymentAdbCompressionAlgorithm)` | `any` | The `adb push -z` compression algorithm. `FastDeploy2` relies on a modern Android SDK Platform-Tools `adb` for multi-file `push -z` support. | -The following internal/unsupported properties tune batching. They exist mainly so -the batching paths can be exercised with smaller batches while testing; their -defaults match the matching task properties: +The following internal/unsupported properties tune or disable implementation +details: | Property | Default | Description | | --- | --- | --- | @@ -33,6 +32,8 @@ defaults match the matching task properties: | `$(_AndroidFastDeployCopyBatchSize)` | `25` | Number of files copied per batch when staging fast-deployment files. | | `$(_AndroidFastDeployMaxShellCommandLength)` | `900` | Maximum length of a single `adb shell` command line before it is split. | | `$(_AndroidFastDeployMaxAdbCommandLength)` | `4096` | Maximum length of a single `adb` command line before it is split. | +| `$(_AndroidFastDeploySkipCleanup)` | blank | Set to `true` to skip orphan staging cleanup. | +| `$(_AndroidFastDeployForceCleanup)` | blank | Set to `true` to bypass the orphan cleanup interval and age guards on a cold deployment. | ## On-device layout @@ -47,6 +48,16 @@ defaults match the matching task properties: deployed manifest so the next build can detect whether the device is already up to date and skip redundant work. +At the start of an install, FastDeploy2 also checks for orphaned staging +directories. The check runs at most once every 24 hours per device. In one +`adb shell` command it enumerates staged `/` directories, +compares them with `pm list packages --user `, and removes directories +for packages that are no longer installed. A staging directory must also be at +least 24 hours old before it can be removed, which prevents cleanup from racing a +concurrent first-time deployment. Cleanup runs only on the cold deployment path, +so warm incremental installs do not perform this check or add another `adb` +invocation. + Changing `$(_AndroidFastDevStrategy)` or `$(_AndroidFastDeployAppFileTransferMode)` invalidates the deployment configuration. In particular, switching from `FastDeploy2` to legacy @@ -196,3 +207,55 @@ Install failures are reported with `ADB####` codes; fast-deployment shell failures (`mkdir`/`rm`/`push`/`ln`) are reported with `XA0129`. `run-as` diagnostics map to `XA0131`–`XA0137`. See the [build/deploy message docs](../docs-mobile/messages/index.md) for details. + +## Command Compatibility + +.NET for Android supports Android 7.0 (API level 24) and later. FastDeploy2's +device-side commands are available by Android 6.0 (API level 23), before the +supported device floor. The API levels below are approximate because shell +utilities are not Android SDK APIs. + +Host-side `adb` commands depend on the installed Android SDK Platform-Tools +version rather than the device API level: + +| Command | FastDeploy2 use | Compatibility | +| --- | --- | --- | +| `adb devices`, `adb -s shell ...` | Device selection and all device-side operations | Standard Platform-Tools commands | +| `adb install -r -d [-t] [--user ]` | APK installation and replacement | Standard Platform-Tools command; `--user` corresponds to Android multi-user support introduced in API 17 | +| `adb push -z ` | Batched compressed staging-file upload | Modern Platform-Tools capability; not controlled by the device application API level | + +FastDeploy2 uses these device-side commands and shell features: + +| Command or shell feature | FastDeploy2 use | Approximate availability | +| --- | --- | --- | +| `sh`/mksh syntax, `[ ... ]`, `test`, `command -v`, `cd`, `pwd`, `echo`, `trap`, globbing, command/parameter/arithmetic expansion, and redirection | Combined checks, override updates, and cleanup control flow | API 14; Android has used mksh since Android 4.0 | +| `getprop` | Validate `run-as` compatibility properties | API 1 | +| `run-as ` | Access the private data directory of a debuggable app | Early Android; availability alone is insufficient because the package must be debuggable and the device must permit `run-as` | +| `su ` | Access files for a system application when adbd is not root | Not guaranteed on production devices; used only for the system-app fallback | +| `cat`, `true`, `rm -f`, `rm -rf`, `mkdir -p`, `rmdir`, `touch`, `cp -p`, and `ln -sf` | Read markers and manage staging/override files, locks, copies, and symlinks | API 21 or earlier; supplied by toolbox/BSD utilities before toybox | +| `readlink -f`, `whoami` | Resolve system-app paths and determine whether adbd is root | Reliably available by API 23 | +| `pidof` | Find the running application process | Reliably available by API 23 | +| `find -type f -exec stat ... {} +` and `stat -c` | Enumerate and compare staged and override files | API 23; supplied by toybox starting in Android 6.0 | +| `printf %s` | Write manifest hash markers without a trailing newline | API 23; supplied by toybox starting in Android 6.0 | +| `date +%s` | Cleanup rate limiting and staging-directory age checks | API 21 or earlier | +| `grep -Fqx` | Exact installed-package lookup during orphan cleanup | API 21 or earlier | +| `pm list packages --user ` | Find installed packages during orphan cleanup | API 17 | +| `pm uninstall [-k] [--user ]` | Remove an incompatible package before retrying installation | Base command predates the supported floor; `--user` requires API 17 multi-user support | +| `am force-stop ` | Stop the app before replacing fast-deployment files | API 8 or earlier | +| `am start-user -w ` | Ensure a secondary Android user is running before `run-as` | API 17 multi-user support | + +The orphan cleanup command additionally checks that each external utility is +present before cleanup. Missing utilities, failed or empty `pm` output, `stat` +failures, and `grep` errors all skip deletion. Only `grep` exit status `1`, +meaning a definite non-match, permits an old staging directory to be removed. + +These estimates are based on the +[AOSP shell and utility inventories][aosp-shell-utilities], the +[Android 6.0 toybox build][aosp-marshmallow-toybox], the +[Android 4.2 `pm --user` implementation][aosp-pm-user], and the +[Android Debug Bridge documentation][adb-docs]. + +[aosp-shell-utilities]: https://android.googlesource.com/platform/system/core/+/refs/heads/main/shell_and_utilities/README.md +[aosp-marshmallow-toybox]: https://android.googlesource.com/platform/external/toybox/+/android-6.0.1_r81/Android.mk +[aosp-pm-user]: https://android.googlesource.com/platform/frameworks/base/+/android-4.2_r1/cmds/pm/src/com/android/commands/pm/Pm.java +[adb-docs]: https://developer.android.com/tools/adb diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.Manifest.cs b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.Manifest.cs index d03b2991186..8edc31ca04f 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.Manifest.cs +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.Manifest.cs @@ -14,9 +14,53 @@ public partial class FastDeploy2 { const string RemoteStagingRootPath = "/data/local/tmp/fastdeploy2"; const string ManifestHashMarker = ".fastdeploy2-manifest-hash"; + const string CleanupMarker = ".last-orphan-cleanup"; + const string CleanupLockDirectory = ".orphan-cleanup-lock"; + const int CleanupIntervalSeconds = 24 * 60 * 60; + const int CleanupSafetyWindowSeconds = 24 * 60 * 60; string RemoteStagingRoot => RemoteStagingRootPath; + async Task CleanupRemoteStagingDirectories (bool force) + { + string command = CreateRemoteStagingCleanupCommand ( + RemoteStagingRoot, + force ? 0 : CleanupIntervalSeconds, + force ? 0 : CleanupSafetyWindowSeconds); + AdbCommandResult result = await RunAdbShellCommand (command); + if (result.ExitCode != 0) { + LogDiagnostic ($"FastDeploy2 orphan staging cleanup failed and will be skipped. Output: {result.Output}"); + } else if (!string.IsNullOrEmpty (result.StandardOutput)) { + LogDiagnostic (result.StandardOutput); + } + } + + static string CreateRemoteStagingCleanupCommand (string remoteStagingRoot, int cleanupIntervalSeconds, int safetyWindowSeconds) + { + return string.Join ("; ", new [] { + $"root={QuoteShellArgument (remoteStagingRoot)}", + $"marker=\"$root/{CleanupMarker}\"", + $"lock=\"$root/{CleanupLockDirectory}\"", + "if [ ! -d \"$root\" ]; then echo 'FastDeploy2 orphan staging cleanup: no staging root'; exit 0; fi", + "for tool in date stat rm mkdir rmdir touch pm grep; do if ! command -v \"$tool\" >/dev/null 2>&1; then echo \"FastDeploy2 orphan staging cleanup: $tool unavailable\"; exit 0; fi; done", + "now=$(date +%s) || exit 1", + "last=$(stat -c %Y \"$marker\" 2>/dev/null || echo 0)", + $"if [ \"$((now - last))\" -lt {cleanupIntervalSeconds} ]; then echo 'FastDeploy2 orphan staging cleanup: already checked'; exit 0; fi", + "lock_time=$(stat -c %Y \"$lock\" 2>/dev/null || echo \"$now\")", + $"if [ -d \"$lock\" ] && [ \"$((now - lock_time))\" -ge {CleanupIntervalSeconds} ]; then rm -rf \"$lock\"; fi", + "if ! mkdir \"$lock\" 2>/dev/null; then echo 'FastDeploy2 orphan staging cleanup: already running'; exit 0; fi", + "trap 'rm -rf \"$lock\"' 0", + "last=$(stat -c %Y \"$marker\" 2>/dev/null || echo 0)", + $"if [ \"$((now - last))\" -lt {cleanupIntervalSeconds} ]; then echo 'FastDeploy2 orphan staging cleanup: already checked'; exit 0; fi", + "touch \"$marker\" || exit 1", + "removed=0", + "status=0", + $"for user_dir in \"$root\"/*/*; do [ -d \"$user_dir\" ] || continue; modified=$(stat -c %Y \"$user_dir\" 2>/dev/null) || {{ status=1; continue; }}; [ \"$((now - modified))\" -ge {safetyWindowSeconds} ] || continue; package_dir=${{user_dir%/*}}; package=${{package_dir##*/}}; user=${{user_dir##*/}}; case \"$user\" in ''|*[!0-9]*) continue ;; esac; packages_file=\"$lock/packages-$user\"; packages_failed=\"$packages_file.failed\"; if [ ! -f \"$packages_file\" ] && [ ! -f \"$packages_failed\" ]; then if ! pm list packages --user \"$user\" > \"$packages_file\" || [ ! -s \"$packages_file\" ]; then rm -f \"$packages_file\"; touch \"$packages_failed\"; status=1; fi; fi; [ -f \"$packages_failed\" ] && continue; grep -Fqx \"package:$package\" \"$packages_file\"; grep_status=$?; if [ \"$grep_status\" -eq 0 ]; then continue; fi; if [ \"$grep_status\" -ne 1 ]; then status=1; continue; fi; if rm -rf \"$user_dir\"; then rmdir \"$package_dir\" 2>/dev/null || true; removed=$((removed + 1)); else status=1; fi; done", + "echo \"FastDeploy2 orphan staging cleanup: removed $removed directories\"", + "exit \"$status\"", + }); + } + async Task DeployFastDevFilesWithAdbPush (string overridePath, bool forceFreshDeployment = false) { var files = PrepareDirectPushFiles (); diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs index 4c9cebb38fb..b963eaa7879 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs @@ -62,6 +62,10 @@ public partial class FastDeploy2 : AsyncTask public bool DiagnosticLogging { get; set; } = false; + public bool FastDeploySkipCleanup { get; set; } = false; + + public bool FastDeployForceCleanup { get; set; } = false; + public string UserID { get; set; } public bool IsTestOnly { get; set; } @@ -185,12 +189,17 @@ public async override Task RunTaskAsync () async Task RunInstall () { - WarmStateProbeOutcome warmState = await TryRunWarmStateProbe (LoadPreviousManifest ()); + ManifestData previousManifest = LoadPreviousManifest (); + WarmStateProbeOutcome warmState = await TryRunWarmStateProbe (previousManifest); if (warmState == WarmStateProbeOutcome.Failed) { return; } if (warmState != WarmStateProbeOutcome.Ready) { + if (!FastDeploySkipCleanup) { + await CleanupRemoteStagingDirectories (force: FastDeployForceCleanup); + } + string redirectStdio = await GetDeviceProperty ("log.redirect-stdio"); if (string.Equals ("true", redirectStdio, StringComparison.OrdinalIgnoreCase)) { LogFastDeploy2Error ("XA0128", Resources.XA0128_RedirectStdioIsEnabled); diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets b/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets index 34ab9814ad6..e2210beaeee 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets @@ -399,6 +399,8 @@ Copyright (C) 2016 Xamarin. All rights reserved. CopyBatchSize="$(_AndroidFastDeployCopyBatchSize)" MaxShellCommandLength="$(_AndroidFastDeployMaxShellCommandLength)" MaxAdbCommandLength="$(_AndroidFastDeployMaxAdbCommandLength)" + FastDeploySkipCleanup="$(_AndroidFastDeploySkipCleanup)" + FastDeployForceCleanup="$(_AndroidFastDeployForceCleanup)" /> diff --git a/tests/MSBuildDeviceIntegration/Tests/FastDevTest.cs b/tests/MSBuildDeviceIntegration/Tests/FastDevTest.cs index 249b9d2bed0..9926979c90c 100644 --- a/tests/MSBuildDeviceIntegration/Tests/FastDevTest.cs +++ b/tests/MSBuildDeviceIntegration/Tests/FastDevTest.cs @@ -321,6 +321,49 @@ public void FastDeploy2RestoresMissingRemoteDirectory () } } + [Test] + public void FastDeploy2CleansOnlyOldOrphanStagingDirectories () + { + string [] oldOrphanDirectories = { + "/data/local/tmp/fastdeploy2/com.xamarin.fastdeploy2_cleanup_old_one/0", + "/data/local/tmp/fastdeploy2/com.xamarin.fastdeploy2_cleanup_old_two/0", + }; + string freshOrphanDirectory = "/data/local/tmp/fastdeploy2/com.xamarin.fastdeploy2_cleanup_fresh/0"; + var proj = new XamarinAndroidApplicationProject { + PackageName = "com.xamarin.fastdeploy2_cleanup", + }; + proj.SetDefaultTargetDevice (); + proj.SetProperty ("_AndroidFastDevStrategy", "FastDeploy2"); + + using (var builder = CreateApkBuilder ()) { + try { + RunAdbCommand ("shell rm -f /data/local/tmp/fastdeploy2/.last-orphan-cleanup"); + foreach (string directory in oldOrphanDirectories) { + RunAdbCommand ($"shell mkdir -p {directory}"); + RunAdbCommand ($"shell touch {directory}/orphan.txt"); + RunAdbCommand ($"shell touch -t 200001010000 {directory}"); + } + RunAdbCommand ($"shell mkdir -p {freshOrphanDirectory}"); + RunAdbCommand ($"shell touch {freshOrphanDirectory}/orphan.txt"); + + Assert.IsTrue (builder.Install (proj), "FastDeploy2 install should have succeeded."); + + foreach (string directory in oldOrphanDirectories) { + Assert.AreEqual ("missing", RunAdbCommand ($"shell if test -e {directory}; then echo exists; else echo missing; fi").Trim (), + $"Old orphan staging directory '{directory}' should have been deleted."); + } + Assert.AreEqual ("exists", RunAdbCommand ($"shell if test -f {freshOrphanDirectory}/orphan.txt; then echo exists; else echo missing; fi").Trim (), + "Default cleanup should preserve a fresh orphan staging file."); + } finally { + foreach (string directory in oldOrphanDirectories) { + RunAdbCommand ($"shell rm -rf {directory}"); + } + RunAdbCommand ($"shell rm -rf {freshOrphanDirectory}"); + builder.Uninstall (proj); + } + } + } + string GetOverrideFileKind (string packageName, string path) { return RunAdbCommand ($"shell run-as {packageName} sh -c 'if test -L {path}; then echo symlink; elif test -f {path}; then echo regular; else echo missing; fi'").Trim ();