diff --git a/.github/workflows/cluster-faces-test.yml b/.github/workflows/cluster-faces-test.yml index e07a43712..3a4a0cc6a 100644 --- a/.github/workflows/cluster-faces-test.yml +++ b/.github/workflows/cluster-faces-test.yml @@ -25,7 +25,7 @@ jobs: # do not stop on another job's failure fail-fast: false matrix: - php-versions: ['8.2'] + php-versions: ['8.3'] databases: ['sqlite'] server-versions: ['master'] pure-js-mode: ['false'] @@ -148,7 +148,7 @@ jobs: id: photos-cache with: path: data/admin/files/ - key: https://cloud.nextcloud.com/public.php/dav/files/wfDk23DBsXYrd4S/IMDb-Face.zip + key: https://cloud.nextcloud.com/public.php/dav/files/wfDk23DBsXYrd4S/IMDb-Face.zip-half - name: Upload photos if: steps.photos-cache.outputs.cache-hit != 'true' @@ -158,11 +158,21 @@ jobs: wget https://cloud.nextcloud.com/public.php/dav/files/wfDk23DBsXYrd4S/IMDb-Face.zip unzip IMDb-Face.zip rm IMDb-Face.zip + # Drop every other identity directory to roughly halve the dataset so + # the classification run stays within the job time limit. Deterministic + # (sorted glob) so both jobs cache an identical set under the same key. + cd IMDb-Face + i=0 + for d in */; do + if [ $((i % 2)) -eq 1 ]; then rm -rf "$d"; fi + i=$((i + 1)) + done + echo "Kept $(ls -d */ | wc -l) identity directories" - uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: data/admin/files/ - key: https://cloud.nextcloud.com/public.php/dav/files/wfDk23DBsXYrd4S/IMDb-Face.zip + key: https://cloud.nextcloud.com/public.php/dav/files/wfDk23DBsXYrd4S/IMDb-Face.zip-half - name: Set config run: | @@ -184,8 +194,11 @@ jobs: env: GITHUB_REF: ${{ github.ref }} run: | + disk_free() { df -h / | awk 'NR==2 {print $4" free ("$5" used)"}'; } + echo "disk before classification: $(disk_free)" ./occ files:scan admin ./occ recognize:classify + echo "disk after classification: $(disk_free)" - uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: @@ -271,7 +284,7 @@ jobs: - name: Download IMDb-Face.csv working-directory: apps/${{ env.APP_NAME }}/tests/res run: | - wget https://cloud.nextcloud.com/public.php/dav/files/wfDk23DBsXYrd4S/IMDb-Face-csv.zip + wget https://cloud.nextcloud.com/public.php/dav/files/wfDk23DBsXYrd4S/IMDb-Face-csv.zip unzip IMDb-Face-csv.zip rm IMDb-Face-csv.zip @@ -282,34 +295,584 @@ jobs: const COLUMN_URL = 5 const COLUMN_RECT = 3 const COLUMN_DIMS = 4 - + + const csv = fs.readFileSync(__dirname + '/apps/recognize/tests/res/IMDb-Face.csv') + .toString('utf8') + .split('\n') + .map(line => line.split(',')) + + // remove csv header + csv.shift() + + const names = [...new Set(csv.map(image => image[COLUMN_NAME])).values()] + + const selectedNames = names.slice(0, 2000) + + const limitedCsv = selectedNames.flatMap(name => { + return csv.filter(line => line[COLUMN_NAME] === name) + }) + + const allDetections = fs.readFileSync(__dirname + '/out.txt').toString('utf8').trim().split('\n').map(line => line.split('|')) + + const json = require(__dirname + '/out.json'); + + const facesByCluster = json + .reduce((acc, face) => { + const clusterId = parseInt(face.href.split('/')[6]); + acc[clusterId] = [...(acc[clusterId] ?? []), face.realpath.split('/')[4]]; + return acc + }, {}); + + const targetFaces = json + .filter(face => { + return limitedCsv + .some(entry => { + if (entry[COLUMN_NAME] === face.realpath.split('/')[4] && entry[COLUMN_URL].split('/').pop() === face.realpath.split('/').pop()) { + let dims = entry[COLUMN_DIMS].split(' ').map(i => parseInt(i)) + dims = {x: dims[1], y: dims[0]} + const rect = entry[COLUMN_RECT].split(' ').map(i => parseInt(i)) + return Math.abs(face['face-detections'][0].x - rect[0] / dims.x) < 0.05 && Math.abs(face['face-detections'][0].y - rect[1] / dims.y) < 0.05 + } + return false + }) + }) + + const targetFacesPerIdentity = targetFaces.reduce((acc, face) => { + const name = face.realpath.split('/')[4] + acc[name] = acc[name] ?? [] + acc[name].push(face) + return acc + },{}) + + const targetFacesByCluster = targetFaces + .reduce((acc, face) => { + const clusterId = parseInt(face.href.split('/')[6]); + acc[clusterId] = [...(acc[clusterId] ?? []), face.realpath.split('/')[4]]; + return acc + }, {}); + + console.log(facesByCluster); + console.log(targetFacesByCluster); + const clusterTargetAccuracies = Object.entries(targetFacesByCluster) + .filter(([clusterId, names]) => names.length > 1) + .map(([clusterId, names]) => + [...new Set(names).values()] + .map(name1 => + names.filter(name2 => name1 === name2).length + ).sort().reverse()[0] / names.length + ); + const clusterAccuracies = Object.entries(facesByCluster) + .map(([clusterId, names]) => + [...new Set(names).values()] + .map(name1 => + names.filter(name2 => name1 === name2).length + ).sort().reverse()[0] / names.length + ); + const clusteredFaces = Object.entries(facesByCluster) + .map(([clusterId, names]) => names.length) + .reduce((acc, val) => acc+val, 0) + const clusteredTargetFaces = Object.entries(targetFacesByCluster) + .map(([clusterId, names]) => names.length) + .reduce((acc, val) => acc+val, 0) + const clusteredTargetFacesByIdentity = Object.entries(targetFacesByCluster) + .map(([clusterId, names]) => + [...new Set(names).values()] + .map(name1 => + [name1, names.filter(name2 => name1 === name2).length] + ).sort(([name1, size1], [name2, size2]) => size1 - size2).reverse()[0] + ) + .filter(([name,size]) => size > 1) + .reduce((acc, [name, size]) => { + acc[name] = (acc[name] ?? 0) + size + return acc + }, Object.fromEntries(Object.entries(targetFacesPerIdentity).map(([key]) => [key, 0]))) + + console.log(targetFacesPerIdentity) + console.log(clusteredTargetFacesByIdentity) + const averageTargetFacesPerIdentity = Object.entries(targetFacesPerIdentity).reduce((acc, [name, detections]) => acc+detections.length, 0) / Object.entries(targetFacesPerIdentity).length + const averageClusteredTargetFacesByIdentity = Object.entries(clusteredTargetFacesByIdentity).reduce((acc, [name, size]) => acc+size, 0) / Object.entries(clusteredTargetFacesByIdentity).length + + const clusteredTargetFacesByIdentityRate = Object.entries(clusteredTargetFacesByIdentity) + .reduce((acc, [name, size]) => acc + size / targetFacesPerIdentity[name].length, 0) / Object.entries(clusteredTargetFacesByIdentity).length + const identitiesWithPhotos = $(find data/admin/files/IMDb-Face -type d ! -empty | wc -l) + const identitiesWithDetections = Object.entries(targetFacesPerIdentity).length + const identitiesWithEnoughDetections = Object.entries(targetFacesPerIdentity).filter(([name, detections]) => detections.length > 1).length + const identitiesWithClusters = Object.entries(clusteredTargetFacesByIdentity).filter(([name, size]) => size > 1).length + const identitiesWithClustersRate = identitiesWithClusters / identitiesWithEnoughDetections + + const detectedFaces = $(sqlite3 data/nextcloud.db "select count(*) from oc_recognize_face_detections where user_id = 'admin';") + const detectedTargetFaces = allDetections.filter(detection => { + if(detection.length < 3) return false + const x = Number(detection[0]) + const y = Number(detection[1]) + const path = detection[2] + return limitedCsv + .some(entry => { + if (entry[COLUMN_NAME] === path.split('/')[2] && entry[COLUMN_URL].split('/').pop().split('.jpg')[0] === path.split('/').pop().split('.jpg')[0]) { + let dims = entry[COLUMN_DIMS].split(' ').map(i => parseInt(i)) + dims = {x: dims[1], y: dims[0]} + const rect = entry[COLUMN_RECT].split(' ').map(i => parseInt(i)) + return Math.abs(x - rect[0] / dims.x) < 0.05 && Math.abs(y - rect[1] / dims.y) < 0.05 + } + return false + }) + }).length + const totalPhotos = $(ls data/admin/files/IMDb-Face/* | wc -l) + const detectedFacesRate = detectedFaces / totalPhotos + const clusteredTargetFacesRate = clusteredTargetFaces / detectedTargetFaces + const clusteredFacesRate = clusteredFaces / detectedFaces + const averageClusterAccuracy = clusterAccuracies.reduce((acc, val) => acc+val, 0)/clusterAccuracies.length + const averageClusterTargetAccuracy = clusterTargetAccuracies.reduce((acc, val) => acc+val, 0)/clusterTargetAccuracies.length + const targettedShitClusterRate = clusterTargetAccuracies.filter((val) => val < 0.5).length/clusterTargetAccuracies.length + const shitClusterRate = clusterAccuracies.filter((val) => val < 0.5).length/clusterAccuracies.length + console.log({ clusterAccuracies }); + console.log({ clusterTargetAccuracies }); + console.log({ totalPhotos }); + console.log({ detectedFaces }); + console.log({ detectedFacesRate }); + console.log({ detectedTargetFaces }); + console.log({ clusteredFaces }); + console.log({ clusteredFacesRate }) + console.log({ clusteredTargetFaces }) + console.log({ clusteredTargetFacesRate }) + console.log({ averageTargetFacesPerIdentity }) + console.log({ averageClusteredTargetFacesByIdentity }) + console.log({ clusteredTargetFacesByIdentityRate }) + console.log({ identitiesWithPhotos }) + console.log({ identitiesWithDetections }) + console.log({ identitiesWithEnoughDetections }) + console.log({ identitiesWithClusters }) + console.log({ identitiesWithClustersRate }) + console.log({ shitClusterRate }) + console.log({ targettedShitClusterRate }) + console.log({ averageClusterAccuracy }) + console.log({ averageClusterTargetAccuracy }) + console.log({ weightedAccuracy: averageClusterAccuracy * clusteredFacesRate }) + console.log({ weightedTargetAccuracy: averageClusterTargetAccuracy * clusteredTargetFacesRate }) + const combinedScore = (averageClusterTargetAccuracy * identitiesWithClustersRate * clusteredTargetFacesByIdentityRate * clusteredTargetFacesRate) ** (1/4) + console.log({ combinedScore, minCombinedScore: 0.6 }) + if (combinedScore < 0.6 || combinedScore > 1.0) { + console.log('Benchmark result: Bad') + process.exit(1) + } else { + console.log('Benchmark result: Good') + } + " + + php-taskprocessing: + runs-on: ubuntu-latest + + strategy: + # do not stop on another job's failure + fail-fast: false + matrix: + php-versions: ['8.3'] + databases: ['sqlite'] + server-versions: ['master'] + + name: Test cluster-faces command via TaskProcessing on ${{ matrix.server-versions }} + + env: + MYSQL_PORT: 4444 + PGSQL_PORT: 4445 + + # recognize_backend ExApp (manual-install deploy daemon) + PYTHONUNBUFFERED: 1 + APP_HOST: 0.0.0.0 + APP_ID: recognize_backend + APP_PORT: 9031 + APP_SECRET: 12345 + NEXTCLOUD_URL: http://localhost:8080 + + services: + mysql: + image: mariadb:10.5 + ports: + - 4444:3306/tcp + env: + MYSQL_ROOT_PASSWORD: rootpassword + options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 5 + postgres: + image: postgres + ports: + - 4445:5432/tcp + env: + POSTGRES_USER: root + POSTGRES_PASSWORD: rootpassword + POSTGRES_DB: nextcloud + options: --health-cmd pg_isready --health-interval 5s --health-timeout 2s --health-retries 5 + + steps: + - name: Checkout server + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: nextcloud/server + ref: ${{ matrix.server-versions }} + + - name: Checkout submodules + shell: bash + run: | + auth_header="$(git config --local --get http.https://github.com/.extraheader)" + git submodule sync --recursive + git -c "http.extraheader=$auth_header" -c protocol.version=2 submodule update --init --force --recursive --depth=1 + + - name: install ssl-cert + if: env.ACT # Skip this on normal GitHub Actions + run: sudo apt update && sudo apt install -y ssl-cert + + - name: Set up php ${{ matrix.php-versions }} + uses: shivammathur/setup-php@cf4cade2721270509d5b1c766ab3549210a39a2a # v2.33.0 + with: + php-version: ${{ matrix.php-versions }} + tools: phpunit + extensions: mbstring, iconv, fileinfo, intl, sqlite, pdo_mysql, pdo_sqlite, pgsql, pdo_pgsql, gd, zip + + - name: Checkout app + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: apps/${{ env.APP_NAME }} + + - name: Read package.json node and npm engines version + uses: skjnldsv/read-package-engines-version-actions@06d6baf7d8f41934ab630e97d9e6c0bc9c9ac5e4 # v3 + id: versions + with: + path: apps/${{ env.APP_NAME }} + fallbackNode: '^12' + fallbackNpm: '^6' + + - name: Set up node ${{ steps.versions.outputs.nodeVersion }} + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: ${{ steps.versions.outputs.nodeVersion }} + + - name: Set up npm ${{ steps.versions.outputs.npmVersion }} + run: npm i -g npm@"${{ steps.versions.outputs.npmVersion }}" + + - name: install make wget unzip + if: env.ACT # Skip this on normal GitHub Actions + run: sudo apt update && sudo apt install -y make wget unzip + + - name: Install app + working-directory: apps/${{ env.APP_NAME }} + run: | + composer install --no-dev + make all + make remove-binaries + make remove-devdeps + + - name: Set up Nextcloud and install app + if: ${{ matrix.databases != 'pgsql'}} + run: | + sleep 25 + mkdir data + ./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$MYSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password + ./occ app:enable -vvv -f ${{ env.APP_NAME }} + # 4 workers by default + composer run serve & + + - name: Set up Nextcloud and install app + if: ${{ matrix.databases == 'pgsql'}} + run: | + sleep 25 + mkdir data + ./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$PGSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password + ./occ app:enable -vvv -f ${{ env.APP_NAME }} + # 4 workers by default + composer run serve & + + - name: Enable SQLite WAL mode + if: ${{ matrix.databases == 'sqlite' }} + run: | + # WAL lets the ExApp's readers and cron's writers proceed concurrently + # instead of serializing on SQLite's single-writer lock, which otherwise + # stalls task scheduling for many minutes per cron run. The mode is + # persisted in the database header, so it survives across connections. + sqlite3 data/nextcloud.db "PRAGMA journal_mode=WAL;" + + - name: Enable app_api + run: ./occ app:enable -vvv -f app_api + + - name: Checkout app + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: nextcloud/viewer + path: apps/viewer + + - name: Install viewer + run: | + ./occ app:enable -vvv viewer + + - name: Remove unnecessary models to make space + run: | + rm -rf apps/recognize/models + + - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + id: photos-cache + with: + path: data/admin/files/ + key: https://cloud.nextcloud.com/public.php/dav/files/wfDk23DBsXYrd4S/IMDb-Face.zip-half + + - name: Upload photos + if: steps.photos-cache.outputs.cache-hit != 'true' + run: | + mkdir -p data/admin/files/ + cd data/admin/files + wget https://cloud.nextcloud.com/public.php/dav/files/wfDk23DBsXYrd4S/IMDb-Face.zip + unzip IMDb-Face.zip + rm IMDb-Face.zip + # Drop every other identity directory to roughly halve the dataset so + # the classification run stays within the job time limit. Deterministic + # (sorted glob) so both jobs cache an identical set under the same key. + cd IMDb-Face + i=0 + for d in */; do + if [ $((i % 2)) -eq 1 ]; then rm -rf "$d"; fi + i=$((i + 1)) + done + echo "Kept $(ls -d */ | wc -l) identity directories" + + - uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: data/admin/files/ + key: https://cloud.nextcloud.com/public.php/dav/files/wfDk23DBsXYrd4S/IMDb-Face.zip-half + + - name: Set config + run: | + ./occ config:app:set --value true recognize faces.enabled + # Don't force API key usage to allow tests to run + ./occ config:app:set --value false recognize require_api_key + # Hand files off to the recognize_backend ExApp via TaskProcessing + # instead of running TensorFlow locally. + ./occ config:app:set --value true recognize taskprocessing.enabled + + - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + id: db-cache + with: + path: data/nextcloud.db + key: ${{ runner.os }}-${{ matrix.server-versions }}-taskprocessing-${{ hashFiles('data/admin/files/**', 'apps/recognize/lib/Classifiers/AbstractTaskProcessingClassifier.php', 'apps/recognize/lib/Classifiers/TaskProcessing/**', 'apps/recognize/lib/TaskProcessing/**', 'apps/recognize/lib/Db/FaceDetectionMapper.php') }} + + # The remaining classification steps only run on a cache miss. When the + # detection database is restored from cache we can skip deploying the + # ExApp entirely and go straight to clustering (pure PHP). + + + - name: Checkout recognize_backend ExApp + if: steps.db-cache.outputs.cache-hit != 'true' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: nextcloud/recognize_backend + path: recognize_backend + + - name: Set up python 3.11 + if: steps.db-cache.outputs.cache-hit != 'true' + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.11' + cache: 'pip' + cache-dependency-path: recognize_backend/requirements.txt + + - name: Read recognize_backend version + if: steps.db-cache.outputs.cache-hit != 'true' + id: backendinfo + uses: skjnldsv/xpath-action@7e6a7c379d0e9abc8acaef43df403ab4fc4f770c # master + with: + filename: recognize_backend/appinfo/info.xml + expression: "/info/version/text()" + + - name: Install ExApp system dependencies + if: steps.db-cache.outputs.cache-hit != 'true' + run: sudo apt update && sudo apt install -y ffmpeg libsndfile1 libgl1 libglib2.0-0 + + - name: Install ExApp requirements + if: steps.db-cache.outputs.cache-hit != 'true' + working-directory: recognize_backend + run: | + # onnxruntime-gpu pulls in several GB of NVIDIA CUDA wheels that fill + # up the runner disk; swap in the CPU build like the full-run test does. + sed -i 's/^onnxruntime-gpu.*/onnxruntime/' requirements.txt + python3 -m pip install --upgrade pip + # The default PyPI torch wheels bundle ~5GB of NVIDIA CUDA libraries and + # blow up the runner disk. Install the CPU-only build first so the torch + # constraints in requirements.txt are already satisfied. + python3 -m pip install --index-url https://download.pytorch.org/whl/cpu torch torchvision torchaudio + python3 -m pip install -r requirements.txt + + - name: Run ExApp + if: steps.db-cache.outputs.cache-hit != 'true' + working-directory: recognize_backend/lib + env: + APP_VERSION: ${{ steps.backendinfo.outputs.result }} + run: | + python3 main.py > "$GITHUB_WORKSPACE/backend_logs" 2>&1 & + + - name: Register ExApp with app_api + if: steps.db-cache.outputs.cache-hit != 'true' + run: | + ./occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http localhost http://localhost:8080 + ./occ app_api:app:register recognize_backend manual_install --json-info "{\"appid\":\"recognize_backend\",\"name\":\"Recognize Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ steps.backendinfo.outputs.result }}\",\"secret\":\"12345\",\"port\":9031,\"scopes\":[\"TASK_PROCESSING\",\"FILES\"],\"system_app\":0}" --force-scopes --wait-finish + + - name: install sqlite3 + if: steps.db-cache.outputs.cache-hit != 'true' && env.ACT # Skip this on normal GitHub Actions + run: sudo apt update && sudo apt install -y sqlite3 + + - name: Run classification via TaskProcessing + if: steps.db-cache.outputs.cache-hit != 'true' + run: | + # Probe available disk so we can tell whether runs die on ENOSPC. + disk_free() { df -h / | awk 'NR==2 {print $4" free ("$5" used)"}'; } + echo "disk before classification: $(disk_free)" + ./occ upgrade # in case server master has new migrations in the meantime + ./occ files:scan admin + # Kick off a full classification run: SchedulerJob -> StorageCrawlJob + # fills the faces queue and ClassifyFacesJob hands each batch to the + # recognize_backend ExApp as a TaskProcessing task. Results are written + # back asynchronously by the TaskResultListener when the ExApp reports. + ./occ recognize:recrawl + # Drive the background jobs by running cron in a loop until the faces + # queue is drained and no crawl/scheduler jobs remain. + for i in $(seq 1 180); do + php cron.php || true + QUEUE=$(sqlite3 data/nextcloud.db "select count(*) from oc_recognize_queue_faces;") + CRAWL=$(sqlite3 data/nextcloud.db "select count(*) from oc_jobs where class like '%StorageCrawlJob' or class like '%SchedulerJob';") + echo "round $i: faces queue=$QUEUE, pending crawl/scheduler jobs=$CRAWL, disk=$(disk_free)" + if [ "$QUEUE" -eq 0 ] && [ "$CRAWL" -eq 0 ] && [ "$i" -gt 3 ]; then break; fi + sleep 10 + done + # Wait for the ExApp to finish processing all scheduled TaskProcessing + # tasks (status 0=unknown, 1=scheduled, 2=running are still pending). + for i in $(seq 1 240); do + PENDING=$(sqlite3 data/nextcloud.db "select count(*) from oc_taskprocessing_tasks where app_id = 'recognize' and status in (0, 1, 2);") + echo "wait $i: pending recognize taskprocessing tasks=$PENDING, disk=$(disk_free)" + if [ "$PENDING" -eq 0 ]; then break; fi + sleep 30 + done + echo "disk after classification: $(disk_free)" + FAILED=$(sqlite3 data/nextcloud.db "select count(*) from oc_taskprocessing_tasks where app_id = 'recognize' and status = 4;") + echo "failed recognize taskprocessing tasks: $FAILED" + + - uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: data/nextcloud.db + key: ${{ steps.db-cache.outputs.cache-primary-key }} + + - name: Reduce space + run: | + for dirname in data/admin/files/IMDb-Face/*; do truncate -s 0 "${dirname}/*"; done + + - name: install sqlite3 + if: env.ACT # Skip this on normal GitHub Actions + run: sudo apt update && sudo apt install -y sqlite3 + + - name: Create detection summary + run: | + sqlite3 data/nextcloud.db "select x, y, path from oc_recognize_face_detections d LEFT JOIN oc_filecache c ON c.fileid = d.file_id where user_id = 'admin' ORDER BY path;" > out.txt + + - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + id: clustering-cache + with: + path: out.json + key: ${{ runner.os }}-taskprocessing-${{ hashFiles('out.txt', 'apps/recognize/lib/Clustering/**', 'apps/recognize/lib/Dav/**', 'apps/recognize/lib/Service/FaceClusterAnalyzer.php', 'apps/recognize/lib/Command/ClusterFaces.php') }} + + - name: Run clustering + if: steps.clustering-cache.outputs.cache-hit != 'true' + run: | + ./occ upgrade # in case server master has new migrations in the meantime + ./occ recognize:cluster-faces -b 10000 + ./occ recognize:cluster-faces -b 10000 + ./occ recognize:cluster-faces -b 10000 + ./occ recognize:cluster-faces -b 10000 + ./occ recognize:cluster-faces -b 10000 + ./occ recognize:cluster-faces -b 10000 + + - name: install python3 python3-pip jq curl + if: steps.clustering-cache.outputs.cache-hit != 'true' && env.ACT # Skip this on normal GitHub Actions + run: sudo apt update && sudo apt install -y python3 python3-pip jq curl + + - name: Install xq + if: steps.clustering-cache.outputs.cache-hit != 'true' + run: | + pip install yq --break-system-packages + + - name: Download face assignments + if: steps.clustering-cache.outputs.cache-hit != 'true' + run: | + curl -u 'admin:password' --request PROPFIND 'http://localhost:8080/remote.php/dav/recognize/admin/faces/' --header 'Depth: 2' --data ' + + + + + + + + + + + + + + + + + ' > out.xml + cat out.xml + + - name: Parse face assignments + if: steps.clustering-cache.outputs.cache-hit != 'true' + run: | + PATH=$PATH:/home/runner/.local/bin + cat out.xml | xq '.["d:multistatus"]["d:response"] | map(select(.["d:href"] | test("faces/.+?/.+?"))) | map({"href": .["d:href"], "realpath": .["d:propstat"][0]["d:prop"]["nc:realpath"], "face-detections": .["d:propstat"][0]["d:prop"]["nc:face-detections"] | fromjson | map({userId, x, y, height, width, clusterId}) })' > out.json + cat out.json + + - uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: out.json + key: ${{ steps.clustering-cache.outputs.cache-primary-key }} + + - name: Download IMDb-Face.csv + working-directory: apps/${{ env.APP_NAME }}/tests/res + run: | + wget https://cloud.nextcloud.com/public.php/dav/files/wfDk23DBsXYrd4S/IMDb-Face-csv.zip + unzip IMDb-Face-csv.zip + rm IMDb-Face-csv.zip + + - name: Analyse face assignments + run: | + node -e " + const COLUMN_NAME = 0 + const COLUMN_URL = 5 + const COLUMN_RECT = 3 + const COLUMN_DIMS = 4 + const csv = fs.readFileSync(__dirname + '/apps/recognize/tests/res/IMDb-Face.csv') .toString('utf8') .split('\n') .map(line => line.split(',')) - + // remove csv header csv.shift() - + const names = [...new Set(csv.map(image => image[COLUMN_NAME])).values()] - + const selectedNames = names.slice(0, 2000) - + const limitedCsv = selectedNames.flatMap(name => { return csv.filter(line => line[COLUMN_NAME] === name) }) - + const allDetections = fs.readFileSync(__dirname + '/out.txt').toString('utf8').trim().split('\n').map(line => line.split('|')) - + const json = require(__dirname + '/out.json'); - + const facesByCluster = json .reduce((acc, face) => { const clusterId = parseInt(face.href.split('/')[6]); acc[clusterId] = [...(acc[clusterId] ?? []), face.realpath.split('/')[4]]; return acc }, {}); - + const targetFaces = json .filter(face => { return limitedCsv @@ -323,21 +886,21 @@ jobs: return false }) }) - + const targetFacesPerIdentity = targetFaces.reduce((acc, face) => { const name = face.realpath.split('/')[4] acc[name] = acc[name] ?? [] acc[name].push(face) - return acc + return acc },{}) - + const targetFacesByCluster = targetFaces .reduce((acc, face) => { const clusterId = parseInt(face.href.split('/')[6]); acc[clusterId] = [...(acc[clusterId] ?? []), face.realpath.split('/')[4]]; return acc }, {}); - + console.log(facesByCluster); console.log(targetFacesByCluster); const clusterTargetAccuracies = Object.entries(targetFacesByCluster) @@ -370,15 +933,15 @@ jobs: ) .filter(([name,size]) => size > 1) .reduce((acc, [name, size]) => { - acc[name] = (acc[name] ?? 0) + size + acc[name] = (acc[name] ?? 0) + size return acc }, Object.fromEntries(Object.entries(targetFacesPerIdentity).map(([key]) => [key, 0]))) - + console.log(targetFacesPerIdentity) console.log(clusteredTargetFacesByIdentity) const averageTargetFacesPerIdentity = Object.entries(targetFacesPerIdentity).reduce((acc, [name, detections]) => acc+detections.length, 0) / Object.entries(targetFacesPerIdentity).length const averageClusteredTargetFacesByIdentity = Object.entries(clusteredTargetFacesByIdentity).reduce((acc, [name, size]) => acc+size, 0) / Object.entries(clusteredTargetFacesByIdentity).length - + const clusteredTargetFacesByIdentityRate = Object.entries(clusteredTargetFacesByIdentity) .reduce((acc, [name, size]) => acc + size / targetFacesPerIdentity[name].length, 0) / Object.entries(clusteredTargetFacesByIdentity).length const identitiesWithPhotos = $(find data/admin/files/IMDb-Face -type d ! -empty | wc -l) @@ -386,7 +949,7 @@ jobs: const identitiesWithEnoughDetections = Object.entries(targetFacesPerIdentity).filter(([name, detections]) => detections.length > 1).length const identitiesWithClusters = Object.entries(clusteredTargetFacesByIdentity).filter(([name, size]) => size > 1).length const identitiesWithClustersRate = identitiesWithClusters / identitiesWithEnoughDetections - + const detectedFaces = $(sqlite3 data/nextcloud.db "select count(*) from oc_recognize_face_detections where user_id = 'admin';") const detectedTargetFaces = allDetections.filter(detection => { if(detection.length < 3) return false @@ -445,3 +1008,11 @@ jobs: console.log('Benchmark result: Good') } " + + - name: Show ExApp logs + if: always() + run: | + echo '---------------- nextcloud.log ----------------' + tail -n 100 data/nextcloud.log || echo "No nextcloud.log" + echo '---------------- recognize_backend logs ----------------' + [ -f backend_logs ] && cat backend_logs || echo "No backend logs" diff --git a/.github/workflows/files-scan-test.yml b/.github/workflows/files-scan-test.yml index cfae8dd55..6f3ce749b 100644 --- a/.github/workflows/files-scan-test.yml +++ b/.github/workflows/files-scan-test.yml @@ -23,7 +23,7 @@ jobs: # do not stop on another job's failure fail-fast: false matrix: - php-versions: ['8.2'] + php-versions: ['8.3'] databases: ['sqlite', 'mysql', 'pgsql'] server-versions: ['master'] diff --git a/.github/workflows/full-run-test.yml b/.github/workflows/full-run-test.yml index 6aca00ff0..a3dc4a24a 100644 --- a/.github/workflows/full-run-test.yml +++ b/.github/workflows/full-run-test.yml @@ -28,7 +28,7 @@ jobs: # do not stop on another job's failure fail-fast: false matrix: - php-versions: ['8.2'] + php-versions: ['8.3'] databases: ['sqlite'] server-versions: ['master'] pure-js-mode: ['false'] @@ -41,7 +41,7 @@ jobs: # test pure-js once - server-versions: master databases: sqlite - php-versions: 8.2 + php-versions: 8.3 pure-js-mode: true imagenet-enabled: true faces-enabled: true @@ -208,3 +208,258 @@ jobs: if: always() run: | cat data/nextcloud.log + + taskprocessing: + runs-on: ubuntu-latest + + strategy: + # do not stop on another job's failure + fail-fast: false + matrix: + php-versions: ['8.3'] + databases: ['sqlite'] + # recognize_backend requires Nextcloud >= 32, so only test against master here + server-versions: ['master'] + imagenet-enabled: ['true'] + faces-enabled: ['true'] + musicnn-enabled: ['true'] + movinet-enabled: ['true'] + + name: Test classify in taskprocessing mode on ${{ matrix.databases }}-${{ matrix.server-versions }} imagenet:${{ matrix.imagenet-enabled }},faces:${{ matrix.faces-enabled }},movinet:${{ matrix.movinet-enabled }},musicnn:${{ matrix.musicnn-enabled }} + + env: + MYSQL_PORT: 4444 + PGSQL_PORT: 4445 + + # recognize_backend ExApp (deployed via AppAPI's manual_install deploy daemon) + PYTHONUNBUFFERED: 1 + APP_HOST: 0.0.0.0 + APP_ID: recognize_backend + APP_PORT: 9031 + APP_SECRET: 12345 + APP_VERSION: 1.0.0 + NEXTCLOUD_URL: http://localhost:8080 + # No NVIDIA driver on the runner; the backend falls back to CPU automatically + COMPUTE_DEVICE: CPU + + timeout-minutes: 90 # model download + CPU inference is slow on the first (uncached) run + + services: + mysql: + image: mariadb:10.5 + ports: + - 4444:3306/tcp + env: + MYSQL_ROOT_PASSWORD: rootpassword + options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 5 + postgres: + image: postgres + ports: + - 4445:5432/tcp + env: + POSTGRES_USER: root + POSTGRES_PASSWORD: rootpassword + POSTGRES_DB: nextcloud + options: --health-cmd pg_isready --health-interval 5s --health-timeout 2s --health-retries 5 + + steps: + - name: Checkout server + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: nextcloud/server + ref: ${{ matrix.server-versions }} + + - name: Checkout submodules + shell: bash + run: | + auth_header="$(git config --local --get http.https://github.com/.extraheader)" + git submodule sync --recursive + git -c "http.extraheader=$auth_header" -c protocol.version=2 submodule update --init --force --recursive --depth=1 + + - name: Set up php ${{ matrix.php-versions }} + uses: shivammathur/setup-php@cf4cade2721270509d5b1c766ab3549210a39a2a # v2.33.0 + with: + php-version: ${{ matrix.php-versions }} + tools: phpunit + extensions: mbstring, iconv, fileinfo, intl, sqlite, pdo_mysql, pdo_sqlite, pgsql, pdo_pgsql, gd, zip + + - name: Checkout app + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: apps/${{ env.APP_NAME }} + + - name: Read package.json node and npm engines version + uses: skjnldsv/read-package-engines-version-actions@06d6baf7d8f41934ab630e97d9e6c0bc9c9ac5e4 # v3 + id: versions + with: + path: apps/${{ env.APP_NAME }} + fallbackNode: '^12' + fallbackNpm: '^6' + + - name: Set up node ${{ steps.versions.outputs.nodeVersion }} + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: ${{ steps.versions.outputs.nodeVersion }} + + - name: Set up npm ${{ steps.versions.outputs.npmVersion }} + run: npm i -g npm@"${{ steps.versions.outputs.npmVersion }}" + + - name: Install app + working-directory: apps/${{ env.APP_NAME }} + run: | + composer install --no-dev + make all + make remove-binaries + rm -rf models # Make it download from github + wget https://github.com/nextcloud/recognize/releases/download/v3.4.0/test-files.zip + unzip test-files.zip -d tests/res/ + + - name: Set up Nextcloud and install app + run: | + sleep 25 + mkdir data + ./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$MYSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password + ./occ app:enable -vvv -f ${{ env.APP_NAME }} + # 4 workers by default + composer run serve & + + - name: Checkout app_api + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: apps/app_api + repository: nextcloud/app_api + ref: ${{ matrix.server-versions == 'master' && 'main' || matrix.server-versions }} + + - name: Enable app_api + run: ./occ app:enable -vvv -f app_api + + - name: Checkout recognize_backend + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: recognize_backend + repository: nextcloud/recognize_backend + ref: main + + - name: Set up python 3.11 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + cache: 'pip' + cache-dependency-path: recognize_backend/requirements.txt + + - name: Install recognize_backend dependencies + working-directory: recognize_backend + run: | + # onnxruntime-gpu needs CUDA libs at import time; swap in the CPU build for the runner + sed -i 's/^onnxruntime-gpu.*/onnxruntime/' requirements.txt + # The default PyPI torch wheels bundle ~5GB of NVIDIA CUDA libraries and + # blow up the runner disk. Install the CPU-only build first so the torch + # constraints in requirements.txt are already satisfied. + pip install --index-url https://download.pytorch.org/whl/cpu torch torchvision torchaudio + pip install -r requirements.txt + + - name: Cache recognize_backend models + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + id: models-cache-restore + with: + path: recognize_backend-persistent_storage/ + key: ${{ runner.os }}-recognize-backend-models-v1 + + - name: Run recognize_backend + working-directory: recognize_backend/lib + env: + APP_PERSISTENT_STORAGE: ${{ github.workspace }}/recognize_backend-persistent_storage + HF_HOME: ${{ github.workspace }}/recognize_backend-persistent_storage/huggingface + INSIGHTFACE_HOME: ${{ github.workspace }}/recognize_backend-persistent_storage/insightface + run: | + mkdir -p "$APP_PERSISTENT_STORAGE" + python3 main.py > "$GITHUB_WORKSPACE/backend_logs" 2>&1 & + + - name: Register recognize_backend with AppAPI + run: | + ./occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http localhost http://localhost:8080 + ./occ app_api:app:register ${{ env.APP_ID }} manual_install --json-info \ + "{\"appid\":\"${{ env.APP_ID }}\",\"name\":\"Recognize Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ env.APP_VERSION }}\",\"secret\":\"${{ env.APP_SECRET }}\",\"port\":${{ env.APP_PORT }},\"scopes\":[\"TASK_PROCESSING\",\"FILES\"],\"system_app\":1}" \ + --force-scopes --wait-finish + + - name: Upload photos + run: | + find apps/${{ env.APP_NAME }}/tests/res/ -type f -exec curl -u 'admin:password' -T "{}" 'http://localhost:8080/remote.php/webdav/' \; + + - name: Enable taskprocessing mode + run: | + ./occ config:app:set --value true recognize taskprocessing.enabled + ./occ config:app:set --value ${{ matrix.imagenet-enabled }} recognize imagenet.enabled + ./occ config:app:set --value ${{ matrix.faces-enabled }} recognize faces.enabled + ./occ config:app:set --value ${{ matrix.musicnn-enabled }} recognize musicnn.enabled + ./occ config:app:set --value ${{ matrix.movinet-enabled }} recognize movinet.enabled + + - name: Schedule classification tasks + env: + GITHUB_REF: ${{ github.ref }} + run: | + ./occ files:scan admin + # recognize:classify does not work in taskprocessing mode; instead let the + # background jobs crawl the storages and schedule TaskProcessing tasks. + ./occ recognize:recrawl + # Run cron a few times so SchedulerJob -> StorageCrawlJob -> Classify*Job run + # in sequence and schedule the TaskProcessing tasks for the uploaded files. + for i in $(seq 1 12); do + php cron.php -v + sleep 30 + done + + - name: Wait for tasks to be processed by recognize_backend + run: | + set -x + # The backend downloads the models on the first run and processes the tasks; + # TaskResultListener applies the results as each task succeeds. + NEXT_WAIT_TIME=0 + DETECTIONS=0 + until [ $NEXT_WAIT_TIME -eq 60 ] || [ "$DETECTIONS" -gt 0 ]; do + php cron.php -v + DETECTIONS=$(sqlite3 data/nextcloud.db "select count(*) from oc_recognize_face_detections;" 2>/dev/null || echo 0) + echo "face detections so far: $DETECTIONS (iteration $NEXT_WAIT_TIME)" + sleep 30 + NEXT_WAIT_TIME=$((NEXT_WAIT_TIME + 1)) + done + # Fail if the backend never produced any results + [ "$DETECTIONS" -gt 0 ] + + - name: Save recognize_backend models cache + if: always() && steps.models-cache-restore.outputs.cache-hit != 'true' + uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: recognize_backend-persistent_storage/ + key: ${{ steps.models-cache-restore.outputs.cache-primary-key }} + + - name: Cluster faces + run: | + ./occ recognize:cluster-faces + + - name: Install cadaver + if: ${{ matrix.faces-enabled }} + run: | + sudo apt -y install cadaver + + - name: Test webdav access + if: ${{ matrix.faces-enabled }} + run: | + cat > ~/.netrc <addServiceListener('OCP\Files\Config\Event\UserMountRemovedEvent', FileListener::class); // it is not fired as of now, Added and Removed events are fired instead in that order // $context->addServiceListener('OCP\Files\Config\Event\UserMountUpdatedEvent', FileListener::class); + + $dispatcher->addServiceListener(TaskSuccessfulEvent::class, TaskResultListener::class); + $dispatcher->addServiceListener(TaskFailedEvent::class, TaskResultListener::class); } public function register(IRegistrationContext $context): void { @@ -53,6 +63,11 @@ public function register(IRegistrationContext $context): void { /** Register $principalBackend for the DAV collection */ $context->registerServiceAlias('principalBackend', Principal::class); + + $context->registerTaskProcessingTaskType(ImageClassificationTaskType::class); + $context->registerTaskProcessingTaskType(VideoClassificationTaskType::class); + $context->registerTaskProcessingTaskType(AudioClassificationTaskType::class); + $context->registerTaskProcessingTaskType(ImageFaceRecognitionTaskType::class); } /** diff --git a/lib/BackgroundJobs/ClassifierJob.php b/lib/BackgroundJobs/ClassifierJob.php index 3551b9f12..94adfe540 100644 --- a/lib/BackgroundJobs/ClassifierJob.php +++ b/lib/BackgroundJobs/ClassifierJob.php @@ -29,7 +29,7 @@ public function __construct( private SettingsService $settingsService, ) { parent::__construct($time); - $this->setInterval(60 * 5); + $this->setInterval(60); $this->setTimeSensitivity(self::TIME_INSENSITIVE); $this->setAllowParallelRuns($settingsService->getSetting('concurrency.enabled') === 'true'); } @@ -38,10 +38,13 @@ public function __construct( * @param array{storageId: int, rootId: int} $argument */ protected function runClassifier(string $model, array $argument): void { - sleep(10); - if ($this->settingsService->getSetting('concurrency.enabled') !== 'true' && $this->anyOtherClassifierJobsRunning()) { - $this->logger->debug('Stalling job '.static::class.' with argument ' . var_export($argument, true) . ' because other classifiers are already reserved'); - return; + $taskProcessingMode = $this->settingsService->getSetting('taskprocessing.enabled') === 'true'; + if (!$taskProcessingMode) { + sleep(10); + if ($this->settingsService->getSetting('concurrency.enabled') !== 'true' && $this->anyOtherClassifierJobsRunning()) { + $this->logger->debug('Stalling job '.static::class.' with argument ' . var_export($argument, true) . ' because other classifiers are already reserved'); + return; + } } $storageId = $argument['storageId']; @@ -53,9 +56,10 @@ protected function runClassifier(string $model, array $argument): void { return; } $this->logger->debug('Classifying files of storage '.$storageId. ' using '.$model); + $batchSize = $taskProcessingMode ? 500 : $this->getBatchSize(); try { - $this->logger->debug('fetching '.$this->getBatchSize().' files from '.$model.' queue'); - $files = $this->queue->getFromQueue($model, $storageId, $rootId, $this->getBatchSize()); + $this->logger->debug('fetching '.$batchSize.' files from '.$model.' queue'); + $files = $this->queue->getFromQueue($model, $storageId, $rootId, $batchSize); } catch (Exception $e) { $this->settingsService->setSetting($model.'.status', 'false'); $this->logger->error('Cannot retrieve items from '.$model.' queue', ['exception' => $e]); diff --git a/lib/BackgroundJobs/ClassifyFacesJob.php b/lib/BackgroundJobs/ClassifyFacesJob.php index 7684df78f..bc7b168ce 100644 --- a/lib/BackgroundJobs/ClassifyFacesJob.php +++ b/lib/BackgroundJobs/ClassifyFacesJob.php @@ -8,6 +8,7 @@ namespace OCA\Recognize\BackgroundJobs; use OCA\Recognize\Classifiers\Images\ClusteringFaceClassifier; +use OCA\Recognize\Classifiers\TaskProcessing\ImageFaceRecognitionClassifier as TaskProcessingFaceClassifier; use OCA\Recognize\Service\Logger; use OCA\Recognize\Service\QueueService; use OCA\Recognize\Service\SettingsService; @@ -20,11 +21,13 @@ final class ClassifyFacesJob extends ClassifierJob { private SettingsService $settingsService; private ClusteringFaceClassifier $faces; + private TaskProcessingFaceClassifier $tpFaces; - public function __construct(ITimeFactory $time, Logger $logger, QueueService $queue, SettingsService $settingsService, ClusteringFaceClassifier $faceClassifier, IUserMountCache $mountCache, IJobList $jobList) { + public function __construct(ITimeFactory $time, Logger $logger, QueueService $queue, SettingsService $settingsService, ClusteringFaceClassifier $faceClassifier, TaskProcessingFaceClassifier $tpFaces, IUserMountCache $mountCache, IJobList $jobList) { parent::__construct($time, $logger, $queue, $mountCache, $jobList, $settingsService); $this->settingsService = $settingsService; $this->faces = $faceClassifier; + $this->tpFaces = $tpFaces; } /** @@ -39,6 +42,10 @@ protected function run($argument): void { * @return void */ protected function classify(array $files) : void { + if ($this->settingsService->getSetting('taskprocessing.enabled') === 'true') { + $this->tpFaces->classify($files); + return; + } $this->faces->classify($files); } diff --git a/lib/BackgroundJobs/ClassifyImagenetJob.php b/lib/BackgroundJobs/ClassifyImagenetJob.php index a3d6b2914..a9b84ec41 100644 --- a/lib/BackgroundJobs/ClassifyImagenetJob.php +++ b/lib/BackgroundJobs/ClassifyImagenetJob.php @@ -8,6 +8,7 @@ namespace OCA\Recognize\BackgroundJobs; use OCA\Recognize\Classifiers\Images\ImagenetClassifier; +use OCA\Recognize\Classifiers\TaskProcessing\ImageClassifier as TaskProcessingImageClassifier; use OCA\Recognize\Service\Logger; use OCA\Recognize\Service\QueueService; use OCA\Recognize\Service\SettingsService; @@ -20,11 +21,13 @@ final class ClassifyImagenetJob extends ClassifierJob { private SettingsService $settingsService; private ImagenetClassifier $imagenet; + private TaskProcessingImageClassifier $tpImage; - public function __construct(ITimeFactory $time, Logger $logger, QueueService $queue, SettingsService $settingsService, ImagenetClassifier $imagenet, IUserMountCache $mountCache, IJobList $jobList) { + public function __construct(ITimeFactory $time, Logger $logger, QueueService $queue, SettingsService $settingsService, ImagenetClassifier $imagenet, TaskProcessingImageClassifier $tpImage, IUserMountCache $mountCache, IJobList $jobList) { parent::__construct($time, $logger, $queue, $mountCache, $jobList, $settingsService); $this->settingsService = $settingsService; $this->imagenet = $imagenet; + $this->tpImage = $tpImage; } /** @@ -39,6 +42,10 @@ protected function run($argument): void { * @return void */ protected function classify(array $files) : void { + if ($this->settingsService->getSetting('taskprocessing.enabled') === 'true') { + $this->tpImage->classify($files); + return; + } $this->imagenet->classify($files); } diff --git a/lib/BackgroundJobs/ClassifyMovinetJob.php b/lib/BackgroundJobs/ClassifyMovinetJob.php index 5c60f6574..f9d0bfee5 100644 --- a/lib/BackgroundJobs/ClassifyMovinetJob.php +++ b/lib/BackgroundJobs/ClassifyMovinetJob.php @@ -7,6 +7,7 @@ declare(strict_types=1); namespace OCA\Recognize\BackgroundJobs; +use OCA\Recognize\Classifiers\TaskProcessing\VideoClassifier as TaskProcessingVideoClassifier; use OCA\Recognize\Classifiers\Video\MovinetClassifier; use OCA\Recognize\Service\Logger; use OCA\Recognize\Service\QueueService; @@ -20,11 +21,13 @@ final class ClassifyMovinetJob extends ClassifierJob { private SettingsService $settingsService; private MovinetClassifier $movinet; + private TaskProcessingVideoClassifier $tpVideo; - public function __construct(ITimeFactory $time, Logger $logger, QueueService $queue, SettingsService $settingsService, MovinetClassifier $movinet, IUserMountCache $mountCache, IJobList $jobList) { + public function __construct(ITimeFactory $time, Logger $logger, QueueService $queue, SettingsService $settingsService, MovinetClassifier $movinet, TaskProcessingVideoClassifier $tpVideo, IUserMountCache $mountCache, IJobList $jobList) { parent::__construct($time, $logger, $queue, $mountCache, $jobList, $settingsService); $this->settingsService = $settingsService; $this->movinet = $movinet; + $this->tpVideo = $tpVideo; } /** @@ -40,6 +43,10 @@ protected function run($argument): void { * @throws \OCA\Recognize\Exception\Exception */ protected function classify(array $files) : void { + if ($this->settingsService->getSetting('taskprocessing.enabled') === 'true') { + $this->tpVideo->classify($files); + return; + } $this->movinet->classify($files); } diff --git a/lib/BackgroundJobs/ClassifyMusicnnJob.php b/lib/BackgroundJobs/ClassifyMusicnnJob.php index 73a00c4a4..2522d96e9 100644 --- a/lib/BackgroundJobs/ClassifyMusicnnJob.php +++ b/lib/BackgroundJobs/ClassifyMusicnnJob.php @@ -8,6 +8,7 @@ namespace OCA\Recognize\BackgroundJobs; use OCA\Recognize\Classifiers\Audio\MusicnnClassifier; +use OCA\Recognize\Classifiers\TaskProcessing\AudioClassifier as TaskProcessingAudioClassifier; use OCA\Recognize\Service\Logger; use OCA\Recognize\Service\QueueService; use OCA\Recognize\Service\SettingsService; @@ -20,11 +21,13 @@ final class ClassifyMusicnnJob extends ClassifierJob { private SettingsService $settingsService; private MusicnnClassifier $musicnn; + private TaskProcessingAudioClassifier $tpAudio; - public function __construct(ITimeFactory $time, Logger $logger, QueueService $queue, SettingsService $settingsService, MusicnnClassifier $musicnn, IUserMountCache $mountCache, IJobList $jobList) { + public function __construct(ITimeFactory $time, Logger $logger, QueueService $queue, SettingsService $settingsService, MusicnnClassifier $musicnn, TaskProcessingAudioClassifier $tpAudio, IUserMountCache $mountCache, IJobList $jobList) { parent::__construct($time, $logger, $queue, $mountCache, $jobList, $settingsService); $this->settingsService = $settingsService; $this->musicnn = $musicnn; + $this->tpAudio = $tpAudio; } /** @@ -39,6 +42,10 @@ protected function run($argument): void { * @return void */ protected function classify(array $files) : void { + if ($this->settingsService->getSetting('taskprocessing.enabled') === 'true') { + $this->tpAudio->classify($files); + return; + } $this->musicnn->classify($files); } diff --git a/lib/Classifiers/AbstractTaskProcessingClassifier.php b/lib/Classifiers/AbstractTaskProcessingClassifier.php new file mode 100644 index 000000000..f6f982f25 --- /dev/null +++ b/lib/Classifiers/AbstractTaskProcessingClassifier.php @@ -0,0 +1,112 @@ + $queueFiles + */ + public function classify(array $queueFiles): void { + if (count($queueFiles) === 0) { + return; + } + + $storageId = $queueFiles[0]->getStorageId(); + $rootId = $queueFiles[0]->getRootId(); + $userId = $this->findUserForStorage($storageId, $rootId); + if ($userId === null) { + $this->logger->warning('No user with access for storage ' . $storageId . '/' . $rootId . ' found; dropping ' . count($queueFiles) . ' files from ' . $this->getModelName() . ' queue'); + $this->dropFromQueue($queueFiles); + return; + } + + $fileIds = array_values(array_unique(array_map(static fn (QueueFile $qf): int => $qf->getFileId(), $queueFiles))); + + $task = new Task( + $this->getTaskTypeId(), + ['input' => $fileIds], + Application::APP_ID, + $userId, + $this->getModelName(), + ); + + try { + $this->taskProcessingManager->scheduleTask($task); + } catch (\Throwable $e) { + // Leave files in the queue so they can be retried on the next job run + $this->logger->error('Failed to schedule ' . $this->getTaskTypeId() . ' task', ['exception' => $e]); + throw new \RuntimeException('Could not schedule ' . $this->getTaskTypeId() . ' task', 0, $e); + } + + /** + * @psalm-suppress PossiblyNullOperand + * @psalm-suppress InvalidOperand + */ + $this->logger->debug('Scheduled ' . $this->getTaskTypeId() . ' task #' . $task->getId() . ' for ' . count($fileIds) . ' files'); + + // Once scheduled, files leave the queue. The TaskResultListener applies results when the task completes. + $this->dropFromQueue($queueFiles); + } + + private function findUserForStorage(int $storageId, int $rootId): ?string { + $mounts = array_values(array_filter( + $this->userMountCache->getMountsForStorageId($storageId), + static fn (ICachedMountInfo $m): bool => $m->getRootId() === $rootId, + )); + if (count($mounts) === 0) { + return null; + } + return $mounts[0]->getUser()->getUID(); + } + + /** + * @param list $queueFiles + */ + private function dropFromQueue(array $queueFiles): void { + try { + $this->queue->removeFilesFromQueue($this->getModelName(), $queueFiles); + } catch (Exception $e) { + $this->logger->warning('Could not remove ' . count($queueFiles) . ' files from ' . $this->getModelName() . ' queue', ['exception' => $e]); + } + } +} diff --git a/lib/Classifiers/TaskProcessing/AudioClassifier.php b/lib/Classifiers/TaskProcessing/AudioClassifier.php new file mode 100644 index 000000000..71c53a0c5 --- /dev/null +++ b/lib/Classifiers/TaskProcessing/AudioClassifier.php @@ -0,0 +1,24 @@ +setName('recognize:classify') - ->setDescription('Classify all files with the current settings in one go (will likely take a long time)') + ->setDescription('Classify all files with the current settings in one go on the terminal (will likely take a long time; doesn\'t work with TaskProcessing mode)') ->addOption('retry', null, InputOption::VALUE_NONE, "Only classify untagged images"); } diff --git a/lib/Command/Recrawl.php b/lib/Command/Recrawl.php index 01990d888..1790972d4 100644 --- a/lib/Command/Recrawl.php +++ b/lib/Command/Recrawl.php @@ -41,7 +41,7 @@ public function __construct(IJobList $jobList, LoggerInterface $logger, QueueSer */ protected function configure() { $this->setName('recognize:recrawl') - ->setDescription('Go through all files again'); + ->setDescription('Trigger a full classification run in the background'); } /** diff --git a/lib/Db/FaceDetectionMapper.php b/lib/Db/FaceDetectionMapper.php index 939205429..75e93b64d 100644 --- a/lib/Db/FaceDetectionMapper.php +++ b/lib/Db/FaceDetectionMapper.php @@ -7,7 +7,9 @@ declare(strict_types=1); namespace OCA\Recognize\Db; +use OCA\Recognize\Classifiers\TaskProcessing\ImageFaceRecognitionClassifier; use OCA\Recognize\Service\FaceClusterAnalyzer; +use OCA\Recognize\Service\SettingsService; use OCP\AppFramework\Db\DoesNotExistException; use OCP\AppFramework\Db\Entity; use OCP\AppFramework\Db\QBMapper; @@ -21,11 +23,20 @@ */ final class FaceDetectionMapper extends QBMapper { private IConfig $config; + private SettingsService $settingsService; - public function __construct(IDBConnection $db, IConfig $config) { + public function __construct(IDBConnection $db, IConfig $config, SettingsService $settingsService) { parent::__construct($db, 'recognize_face_detections', FaceDetection::class); $this->db = $db; $this->config = $config; + $this->settingsService = $settingsService; + } + + private function getMinDetectionSize(): float { + if ($this->settingsService->getSetting('taskprocessing.enabled') === 'true') { + return ImageFaceRecognitionClassifier::MIN_DETECTION_SIZE; + } + return FaceClusterAnalyzer::MIN_DETECTION_SIZE; } /** @@ -329,12 +340,13 @@ public function findDetectionForPreviewImageByClusterId(int $clusterId) : FaceDe } public function countUnclustered(): int { + $minDetectionSize = $this->getMinDetectionSize(); $qb = $this->db->getQueryBuilder(); $qb->select($qb->func()->count('id')) ->from('recognize_face_detections') ->where($qb->expr()->isNull('cluster_id')) - ->andWhere($qb->expr()->gte('height', $qb->createPositionalParameter(FaceClusterAnalyzer::MIN_DETECTION_SIZE))) - ->andWhere($qb->expr()->gte('width', $qb->createPositionalParameter(FaceClusterAnalyzer::MIN_DETECTION_SIZE))); + ->andWhere($qb->expr()->gte('height', $qb->createPositionalParameter($minDetectionSize))) + ->andWhere($qb->expr()->gte('width', $qb->createPositionalParameter($minDetectionSize))); $result = $qb->executeQuery(); /** @var int|string $count */ $count = $result->fetch(\PDO::FETCH_COLUMN); @@ -347,12 +359,13 @@ public function countUnclustered(): int { * @throws \OCP\DB\Exception */ public function getUsersForUnclustered(): array { + $minDetectionSize = $this->getMinDetectionSize(); $qb = $this->db->getQueryBuilder(); $qb->selectDistinct('user_id') ->from('recognize_face_detections') ->where($qb->expr()->isNull('cluster_id')) - ->andWhere($qb->expr()->gte('height', $qb->createPositionalParameter(FaceClusterAnalyzer::MIN_DETECTION_SIZE))) - ->andWhere($qb->expr()->gte('width', $qb->createPositionalParameter(FaceClusterAnalyzer::MIN_DETECTION_SIZE))); + ->andWhere($qb->expr()->gte('height', $qb->createPositionalParameter($minDetectionSize))) + ->andWhere($qb->expr()->gte('width', $qb->createPositionalParameter($minDetectionSize))); $result = $qb->executeQuery(); /** @var array $users */ $users = $result->fetchAll(\PDO::FETCH_COLUMN); diff --git a/lib/Db/QueueMapper.php b/lib/Db/QueueMapper.php index 12b0da50f..22117ca51 100644 --- a/lib/Db/QueueMapper.php +++ b/lib/Db/QueueMapper.php @@ -79,6 +79,31 @@ public function removeFromQueue(string $model, QueueFile $file) : void { ->executeStatement(); } + /** + * Remove multiple queue items in a single statement per chunk. + * + * Deleting files one-by-one issues one write transaction per file, which on + * SQLite serializes against every other DB user and can stall for minutes + * when a large batch is dropped. Batching keeps it to a handful of writes. + * + * @param string $model + * @param list $ids + * @return void + * @throws \OCP\DB\Exception + */ + public function removeFromQueueByIds(string $model, array $ids) : void { + if (count($ids) === 0) { + return; + } + // Chunk to stay well below SQLite's bound-parameter limit + foreach (array_chunk($ids, 500) as $chunk) { + $qb = $this->db->getQueryBuilder(); + $qb->delete($this->getQueueTable($model)) + ->where($qb->expr()->in('id', $qb->createNamedParameter($chunk, IQueryBuilder::PARAM_INT_ARRAY))) + ->executeStatement(); + } + } + /** * @param int $fileId * @return void diff --git a/lib/Service/FaceClusterAnalyzer.php b/lib/Service/FaceClusterAnalyzer.php index 5c8e01052..9d1b90736 100644 --- a/lib/Service/FaceClusterAnalyzer.php +++ b/lib/Service/FaceClusterAnalyzer.php @@ -9,6 +9,7 @@ use \OCA\Recognize\Vendor\Rubix\ML\Datasets\Labeled; use \OCA\Recognize\Vendor\Rubix\ML\Kernels\Distance\Euclidean; +use OCA\Recognize\Classifiers\TaskProcessing\ImageFaceRecognitionClassifier; use OCA\Recognize\Clustering\HDBSCAN; use OCA\Recognize\Db\FaceCluster; use OCA\Recognize\Db\FaceClusterMapper; @@ -27,7 +28,12 @@ final class FaceClusterAnalyzer { private FaceDetectionMapper $faceDetections; private FaceClusterMapper $faceClusters; private Logger $logger; - private int $minDatasetSize = self::MIN_DATASET_SIZE; + private int $minDatasetSize; + private float $minDetectionSize; + private float $minClusterSeparation; + private float $maxClusterEdgeLength; + private float $maxOverlapNewCluster; + private float $minOverlapExistingCluster; private SettingsService $settingsService; public function __construct(FaceDetectionMapper $faceDetections, FaceClusterMapper $faceClusters, Logger $logger, SettingsService $settingsService) { @@ -35,6 +41,22 @@ public function __construct(FaceDetectionMapper $faceDetections, FaceClusterMapp $this->faceClusters = $faceClusters; $this->logger = $logger; $this->settingsService = $settingsService; + + if ($this->settingsService->getSetting('taskprocessing.enabled') === 'true') { + $this->minDatasetSize = ImageFaceRecognitionClassifier::MIN_DATASET_SIZE; + $this->minDetectionSize = ImageFaceRecognitionClassifier::MIN_DETECTION_SIZE; + $this->minClusterSeparation = ImageFaceRecognitionClassifier::MIN_CLUSTER_SEPARATION; + $this->maxClusterEdgeLength = ImageFaceRecognitionClassifier::MAX_CLUSTER_EDGE_LENGTH; + $this->maxOverlapNewCluster = ImageFaceRecognitionClassifier::MAX_OVERLAP_NEW_CLUSTER; + $this->minOverlapExistingCluster = ImageFaceRecognitionClassifier::MIN_OVERLAP_EXISTING_CLUSTER; + } else { + $this->minDatasetSize = self::MIN_DATASET_SIZE; + $this->minDetectionSize = self::MIN_DETECTION_SIZE; + $this->minClusterSeparation = self::MIN_CLUSTER_SEPARATION; + $this->maxClusterEdgeLength = self::MAX_CLUSTER_EDGE_LENGTH; + $this->maxOverlapNewCluster = self::MAX_OVERLAP_NEW_CLUSTER; + $this->minOverlapExistingCluster = self::MIN_OVERLAP_EXISTING_CLUSTER; + } } public function setMinDatasetSize(int $minSize) : void { @@ -64,12 +86,12 @@ public function calculateClusters(string $userId, int $batchSize = 0): void { } if ($batchSize > 0) { - $rejectedDetections = $this->faceDetections->sampleRejectedDetectionsByUserId($userId, $this->getRejectSampleSize($batchSize), self::MIN_DETECTION_SIZE, self::MIN_DETECTION_SIZE); + $rejectedDetections = $this->faceDetections->sampleRejectedDetectionsByUserId($userId, $this->getRejectSampleSize($batchSize), $this->minDetectionSize, $this->minDetectionSize); $requestedFreshDetectionCount = max($batchSize - count($rejectedDetections) - count($sampledDetections), 500); - $freshDetections = $this->faceDetections->findUnclusteredByUserId($userId, $requestedFreshDetectionCount, self::MIN_DETECTION_SIZE, self::MIN_DETECTION_SIZE); + $freshDetections = $this->faceDetections->findUnclusteredByUserId($userId, $requestedFreshDetectionCount, $this->minDetectionSize, $this->minDetectionSize); } else { - $freshDetections = $this->faceDetections->findUnclusteredByUserId($userId, 0, self::MIN_DETECTION_SIZE, self::MIN_DETECTION_SIZE); - $rejectedDetections = $this->faceDetections->sampleRejectedDetectionsByUserId($userId, $this->getRejectSampleSize(count($freshDetections)), self::MIN_DETECTION_SIZE, self::MIN_DETECTION_SIZE); + $freshDetections = $this->faceDetections->findUnclusteredByUserId($userId, 0, $this->minDetectionSize, $this->minDetectionSize); + $rejectedDetections = $this->faceDetections->sampleRejectedDetectionsByUserId($userId, $this->getRejectSampleSize(count($freshDetections)), $this->minDetectionSize, $this->minDetectionSize); } @@ -94,7 +116,7 @@ public function calculateClusters(string $userId, int $batchSize = 0): void { $hdbscan = new HDBSCAN($dataset, $this->getMinClusterSize($n), $this->getMinSampleSize($n)); $numberOfClusteredDetections = 0; - $clusters = $hdbscan->predict(self::MIN_CLUSTER_SEPARATION, self::MAX_CLUSTER_EDGE_LENGTH); + $clusters = $hdbscan->predict($this->minClusterSeparation, $this->maxClusterEdgeLength); foreach ($clusters as $flatCluster) { /** @var int[] $detectionKeys */ @@ -132,10 +154,10 @@ public function calculateClusters(string $userId, int $batchSize = 0): void { } // If more than X% of already clustered detections are for this, we keep it - if ($overlap > self::MIN_OVERLAP_EXISTING_CLUSTER) { + if ($overlap > $this->minOverlapExistingCluster) { $clusterId = $oldClusterId; $cluster = $this->faceClusters->find($clusterId); - } elseif ($overlap < self::MAX_OVERLAP_NEW_CLUSTER) { + } elseif ($overlap < $this->maxOverlapNewCluster) { // otherwise we create a new cluster $cluster = new FaceCluster(); @@ -187,17 +209,21 @@ public function calculateClusters(string $userId, int $batchSize = 0): void { * @return list */ public static function calculateCentroidOfDetections(array $detections): array { - // init 128 dimensional vector - /** @var list $sum */ - $sum = []; - for ($i = 0; $i < self::DIMENSIONS; $i++) { - $sum[] = 0.0; - } - if (count($detections) === 0) { - return $sum; + /** @var list $empty */ + $empty = []; + for ($i = 0; $i < self::DIMENSIONS; $i++) { + $empty[] = 0.0; + } + return $empty; } + // Size the accumulator from the first detection so both 128-dim (legacy) and + // 512-dim (buffalo_l/taskprocessing) embeddings work without a runtime switch. + $dimensions = count(reset($detections)->getVector()); + /** @var list $sum */ + $sum = array_fill(0, $dimensions, 0.0); + foreach ($detections as $detection) { $sum = array_map(static function (float $el, float $el2): float { return $el + $el2; diff --git a/lib/Service/QueueService.php b/lib/Service/QueueService.php index 6a624a582..bffee7e75 100644 --- a/lib/Service/QueueService.php +++ b/lib/Service/QueueService.php @@ -102,6 +102,22 @@ public function removeFromQueue(string $model, QueueFile $queueFile) : void { $this->queueMapper->removeFromQueue($model, $queueFile); } + /** + * Remove a batch of queue files in as few statements as possible. + * + * @param string $model + * @param list<\OCA\Recognize\Db\QueueFile> $queueFiles + * @return void + * @throws \OCP\DB\Exception + */ + public function removeFilesFromQueue(string $model, array $queueFiles) : void { + $ids = array_map( + static fn (QueueFile $qf): int => $qf->getId(), + $queueFiles, + ); + $this->queueMapper->removeFromQueueByIds($model, $ids); + } + /** * @throws \OCP\DB\Exception */ diff --git a/lib/Service/SettingsService.php b/lib/Service/SettingsService.php index 93c6eac2f..8ea9d217b 100644 --- a/lib/Service/SettingsService.php +++ b/lib/Service/SettingsService.php @@ -18,8 +18,15 @@ use OCA\Recognize\Exception\Exception; use OCP\AppFramework\Services\IAppConfig; use OCP\BackgroundJob\IJobList; +use OCP\Server; final class SettingsService { + /** + * App id of the ExApp that provides TaskProcessing classifiers; when installed + * and enabled, taskprocessing mode is on by default. + */ + public const RECOGNIZE_BACKEND_APP_ID = 'recognize_backend'; + /** @var array */ private const DEFAULTS = [ 'tensorflow.cores' => '0', @@ -31,6 +38,7 @@ final class SettingsService { 'faces.enabled' => 'false', 'musicnn.enabled' => 'false', 'movinet.enabled' => 'false', + 'taskprocessing.enabled' => '', 'node_binary' => '', 'clusterFaces.status' => 'null', 'faces.status' => 'null', @@ -105,6 +113,10 @@ public function getSetting(string $key): string { if (strpos($key, 'batchSize') !== false) { return $this->config->getAppValueString($key, $this->getSetting('tensorflow.purejs') === 'false' ? self::DEFAULTS[$key] : self::PUREJS_DEFAULTS[$key]); } + if ($key === 'taskprocessing.enabled') { + $default = $this->isRecognizeBackendInstalled() ? 'true' : 'false'; + return $this->config->getAppValueString($key, $default); + } $lazy = false; if (in_array($key, self::LAZY_SETTINGS, true)) { $lazy = true; @@ -112,6 +124,28 @@ public function getSetting(string $key): string { return $this->config->getAppValueString($key, self::DEFAULTS[$key], lazy: $lazy); } + /** + * Whether the recognize_backend ExApp is installed and enabled. The lookup + * goes through app_api's PublicFunctions service so we don't impose a hard + * dependency on app_api: if it isn't installed, this returns false. + */ + public function isRecognizeBackendInstalled(): bool { + try { + /** + * @var \OCA\AppAPI\PublicFunctions $publicFunctions + */ + $publicFunctions = Server::get(\OCA\AppAPI\PublicFunctions::class); + } catch (\Throwable $e) { + return false; + } + try { + $exApp = $publicFunctions->getExApp(self::RECOGNIZE_BACKEND_APP_ID); + } catch (\Throwable $e) { + return false; + } + return $exApp !== null && (bool)($exApp['enabled'] ?? false); + } + /** * @param string $key * @param string $value diff --git a/lib/Settings/AdminSettings.php b/lib/Settings/AdminSettings.php index 83f8a7657..31f133706 100644 --- a/lib/Settings/AdminSettings.php +++ b/lib/Settings/AdminSettings.php @@ -35,6 +35,8 @@ public function getForm(): TemplateResponse { $tagsEnabled = $this->appManager->isEnabledForAnyone('systemtags'); $this->initialState->provideInitialState('tagsEnabled', $tagsEnabled); + $this->initialState->provideInitialState('recognizeBackendInstalled', $this->settingsService->isRecognizeBackendInstalled()); + return new TemplateResponse('recognize', 'admin'); } diff --git a/lib/TaskProcessing/AudioClassificationTaskType.php b/lib/TaskProcessing/AudioClassificationTaskType.php new file mode 100644 index 000000000..8d348c920 --- /dev/null +++ b/lib/TaskProcessing/AudioClassificationTaskType.php @@ -0,0 +1,72 @@ +l->t('Audio classification'); + } + + /** + * @inheritDoc + */ + public function getDescription(): string { + return $this->l->t('Classify audios into categories.'); + } + + /** + * @return string + */ + public function getId(): string { + return self::ID; + } + + /** + * @return ShapeDescriptor[] + */ + public function getInputShape(): array { + return [ + 'input' => new ShapeDescriptor( + $this->l->t('Audios'), + $this->l->t('Provide audios to classify'), + EShapeType::ListOfAudios, + ), + ]; + } + + /** + * @return ShapeDescriptor[] + */ + public function getOutputShape(): array { + return [ + 'output' => new ShapeDescriptor( + $this->l->t('Categories'), + $this->l->t('The classified categories. Each input audio is mapped to a text containing a comma separated list of categories.'), + EShapeType::ListOfTexts, + ), + ]; + } +} diff --git a/lib/TaskProcessing/ImageClassificationTaskType.php b/lib/TaskProcessing/ImageClassificationTaskType.php new file mode 100644 index 000000000..83bc938b0 --- /dev/null +++ b/lib/TaskProcessing/ImageClassificationTaskType.php @@ -0,0 +1,72 @@ +l->t('Image classification'); + } + + /** + * @inheritDoc + */ + public function getDescription(): string { + return $this->l->t('Classify images into categories.'); + } + + /** + * @return string + */ + public function getId(): string { + return self::ID; + } + + /** + * @return ShapeDescriptor[] + */ + public function getInputShape(): array { + return [ + 'input' => new ShapeDescriptor( + $this->l->t('Images'), + $this->l->t('Provide images to classify'), + EShapeType::ListOfImages, + ), + ]; + } + + /** + * @return ShapeDescriptor[] + */ + public function getOutputShape(): array { + return [ + 'output' => new ShapeDescriptor( + $this->l->t('Categories'), + $this->l->t('The classified categories. Each input image is mapped to a text containing a comma separated list of categories.'), + EShapeType::ListOfTexts, + ), + ]; + } +} diff --git a/lib/TaskProcessing/ImageFaceRecognitionTaskType.php b/lib/TaskProcessing/ImageFaceRecognitionTaskType.php new file mode 100644 index 000000000..fefb2f129 --- /dev/null +++ b/lib/TaskProcessing/ImageFaceRecognitionTaskType.php @@ -0,0 +1,72 @@ +l->t('Image face recognition'); + } + + /** + * @inheritDoc + */ + public function getDescription(): string { + return $this->l->t('Recognize faces in images and return embedding vectors for each face.'); + } + + /** + * @return string + */ + public function getId(): string { + return self::ID; + } + + /** + * @return ShapeDescriptor[] + */ + public function getInputShape(): array { + return [ + 'input' => new ShapeDescriptor( + $this->l->t('Images'), + $this->l->t('Provide images to recognize faces in'), + EShapeType::ListOfImages, + ), + ]; + } + + /** + * @return ShapeDescriptor[] + */ + public function getOutputShape(): array { + return [ + 'output' => new ShapeDescriptor( + $this->l->t('Faces'), + $this->l->t('The detected faces. Each input image is mapped to a text containing JSON-encoded face descriptions ({x,y,width,height,score,vector,angle} ) separated by line breaks.'), + EShapeType::ListOfTexts, + ), + ]; + } +} diff --git a/lib/TaskProcessing/TaskResultListener.php b/lib/TaskProcessing/TaskResultListener.php new file mode 100644 index 000000000..bfcad9f6f --- /dev/null +++ b/lib/TaskProcessing/TaskResultListener.php @@ -0,0 +1,287 @@ + + */ +final class TaskResultListener implements IEventListener { + public function __construct( + private LoggerInterface $logger, + private TagManager $tagManager, + private FaceDetectionMapper $faceDetections, + private IUserMountCache $userMountCache, + private IAppConfig $config, + private IJobList $jobList, + private QueueService $queue, + private IUserSession $userSession, + private IUserManager $userManager, + ) { + } + + public function handle(Event $event): void { + if ($event instanceof TaskFailedEvent) { + $this->handleFailure($event); + return; + } + if ($event instanceof TaskSuccessfulEvent) { + $this->handleSuccess($event); + } + } + + private function handleFailure(TaskFailedEvent $event): void { + $task = $event->getTask(); + if (!$this->isOwnTask($task)) { + return; + } + $model = $this->modelForTaskType($task->getTaskTypeId()); + /** + * @psalm-suppress PossiblyNullOperand + * @psalm-suppress InvalidOperand + */ + $this->logger->warning('TaskProcessing task ' . $task->getTaskTypeId() . ' (id=' . $task->getId() . ') failed: ' . $event->getErrorMessage()); + if ($model !== null) { + $this->config->setAppValueString($model . '.status', 'false'); + } + } + + private function handleSuccess(TaskSuccessfulEvent $event): void { + $task = $event->getTask(); + if (!$this->isOwnTask($task)) { + return; + } + + $input = $task->getInput()['input'] ?? null; + $output = ($task->getOutput() ?? [])['output'] ?? null; + if (!is_array($input) || !is_array($output)) { + /** + * @psalm-suppress PossiblyNullOperand + * @psalm-suppress InvalidOperand + */ + $this->logger->warning('TaskProcessing task ' . $task->getTaskTypeId() . ' (id=' . $task->getId() . ') has unexpected input/output shape'); + return; + } + + /** @psalm-suppress RedundantFunctionCallGivenDocblockType */ + $fileIds = array_map('intval', array_values($input)); + /** @psalm-suppress RedundantFunctionCallGivenDocblockType */ + $results = array_values($output); + + $userId = $task->getUserId(); + if ($userId === null) { + /** + * @psalm-suppress PossiblyNullOperand + * @psalm-suppress InvalidOperand + */ + $this->logger->warning('TaskProcessing task ' . $task->getTaskTypeId() . ' (id=' . $task->getId() . ') has no user set, skipping this task'); + return; + } + $this->userSession->setUser($this->userManager->get($userId)); + + switch ($task->getTaskTypeId()) { + case ImageClassificationTaskType::ID: + $this->applyTagResults($fileIds, $results, ImagenetClassifier::MODEL_NAME, false); + break; + case VideoClassificationTaskType::ID: + $this->applyTagResults($fileIds, $results, MovinetClassifier::MODEL_NAME, false); + break; + case AudioClassificationTaskType::ID: + $this->applyTagResults($fileIds, $results, MusicnnClassifier::MODEL_NAME, false); + break; + case ImageFaceRecognitionTaskType::ID: + $this->applyFaceResults($fileIds, $results); + break; + default: + return; + } + } + + private function isOwnTask(Task $task): bool { + return $task->getAppId() === Application::APP_ID + && in_array($task->getTaskTypeId(), [ + ImageClassificationTaskType::ID, + VideoClassificationTaskType::ID, + AudioClassificationTaskType::ID, + ImageFaceRecognitionTaskType::ID, + ], true); + } + + private function modelForTaskType(string $taskTypeId): ?string { + return match ($taskTypeId) { + ImageClassificationTaskType::ID => ImagenetClassifier::MODEL_NAME, + VideoClassificationTaskType::ID => MovinetClassifier::MODEL_NAME, + AudioClassificationTaskType::ID => MusicnnClassifier::MODEL_NAME, + ImageFaceRecognitionTaskType::ID => ClusteringFaceClassifier::MODEL_NAME, + default => null, + }; + } + + /** + * @param list $fileIds + * @param list $results + */ + private function applyTagResults(array $fileIds, array $results, string $model, bool $forwardToLandmarks): void { + foreach ($fileIds as $i => $fileId) { + if (!isset($results[$i])) { + continue; + } + $raw = (string)$results[$i]; + $tags = array_values(array_filter(array_map('trim', explode(',', $raw)), static fn (string $t): bool => $t !== '')); + if (count($tags) === 0) { + $this->logger->debug('No tags returned for file ' . $fileId . ' from ' . $model); + continue; + } + try { + $this->tagManager->assignTags($fileId, $tags); + } catch (\Throwable $e) { + $this->logger->warning('Could not assign ' . $model . ' tags for file ' . $fileId, ['exception' => $e]); + continue; + } + $this->config->setAppValueString($model . '.status', 'true', lazy: true); + $this->config->setAppValueString($model . '.lastFile', (string)time(), lazy: true); + + if ($forwardToLandmarks) { + $landmarkTags = array_values(array_filter($tags, static fn (string $tag): bool => in_array($tag, LandmarksClassifier::PRECONDITION_TAGS, true))); + if (count($landmarkTags) > 0) { + $this->enqueueForLandmarks($fileId); + } + } + } + } + + /** + * @param list $fileIds + * @param list $results + */ + private function applyFaceResults(array $fileIds, array $results): void { + $model = ClusteringFaceClassifier::MODEL_NAME; + $scheduledClusterJobsFor = []; + foreach ($fileIds as $i => $fileId) { + if (!isset($results[$i])) { + continue; + } + $raw = (string)$results[$i]; + $userIds = $this->getUsersWithFileAccess($fileId); + foreach (explode("\n", $raw) as $line) { + $line = trim($line); + if ($line === '') { + continue; + } + try { + $face = json_decode($line, true, 512, JSON_THROW_ON_ERROR); + } catch (\JsonException $e) { + $this->logger->warning('Invalid face JSON for file ' . $fileId, ['exception' => $e]); + continue; + } + if (!is_array($face)) { + continue; + } + if (isset($face['score']) && (float)$face['score'] < ImageFaceRecognitionClassifier::MIN_FACE_RECOGNITION_SCORE) { + continue; + } + // Accept either a full face object {x,y,width,height,score,vector,angle} + // or a bare embedding vector (list of numbers). + $isBareVector = array_is_list($face) && count($face) > 0 && is_numeric($face[0]); + $vector = $isBareVector ? $face : ($face['vector'] ?? null); + if (!is_array($vector)) { + $this->logger->warning('Face entry without embedding vector for file ' . $fileId); + continue; + } + foreach ($userIds as $userId) { + $detection = new FaceDetection(); + $detection->setFileId($fileId); + $detection->setUserId($userId); + $detection->setX((float)($face['x'] ?? 0)); + $detection->setY((float)($face['y'] ?? 0)); + $detection->setWidth((float)($face['width'] ?? 0)); + $detection->setHeight((float)($face['height'] ?? 0)); + $detection->setVector($vector); + try { + $this->faceDetections->insert($detection); + } catch (\Throwable $e) { + $this->logger->error('Could not store face detection for file ' . $fileId, ['exception' => $e]); + continue; + } + if (!isset($scheduledClusterJobsFor[$userId])) { + $this->jobList->add(ClusterFacesJob::class, ['userId' => $userId]); + $scheduledClusterJobsFor[$userId] = true; + } + } + } + $this->config->setAppValueString($model . '.status', 'true', lazy: true); + $this->config->setAppValueString($model . '.lastFile', (string)time(), lazy: true); + } + } + + private function enqueueForLandmarks(int $fileId): void { + $mounts = $this->userMountCache->getMountsForFileId($fileId); + if (count($mounts) === 0) { + return; + } + $mount = $mounts[0]; + $queueFile = new QueueFile(); + $queueFile->setFileId($fileId); + $queueFile->setStorageId($mount->getStorageId()); + $queueFile->setRootId($mount->getRootId()); + $queueFile->setUpdate(false); + try { + $this->queue->insertIntoQueue(LandmarksClassifier::MODEL_NAME, $queueFile); + } catch (\Throwable $e) { + $this->logger->warning('Could not enqueue file ' . $fileId . ' for landmark detection', ['exception' => $e]); + } + } + + /** + * @return list + */ + private function getUsersWithFileAccess(int $fileId): array { + try { + $mountInfos = $this->userMountCache->getMountsForFileId($fileId); + } catch (\Throwable $e) { + $this->logger->warning('Could not look up users with access for file ' . $fileId, ['exception' => $e]); + return []; + } + return array_values(array_unique(array_map( + static fn (ICachedMountInfo $m): string => $m->getUser()->getUID(), + $mountInfos, + ))); + } +} diff --git a/lib/TaskProcessing/VideoClassificationTaskType.php b/lib/TaskProcessing/VideoClassificationTaskType.php new file mode 100644 index 000000000..0f8203099 --- /dev/null +++ b/lib/TaskProcessing/VideoClassificationTaskType.php @@ -0,0 +1,72 @@ +l->t('Video classification'); + } + + /** + * @inheritDoc + */ + public function getDescription(): string { + return $this->l->t('Classify videos into categories.'); + } + + /** + * @return string + */ + public function getId(): string { + return self::ID; + } + + /** + * @return ShapeDescriptor[] + */ + public function getInputShape(): array { + return [ + 'input' => new ShapeDescriptor( + $this->l->t('Videos'), + $this->l->t('Provide videos to classify'), + EShapeType::ListOfVideos, + ), + ]; + } + + /** + * @return ShapeDescriptor[] + */ + public function getOutputShape(): array { + return [ + 'output' => new ShapeDescriptor( + $this->l->t('Categories'), + $this->l->t('The classified categories. Each input video is mapped to a text containing a comma separated list of categories.'), + EShapeType::ListOfTexts, + ), + ]; + } +} diff --git a/psalm.xml b/psalm.xml index a7f8113a9..72891004f 100644 --- a/psalm.xml +++ b/psalm.xml @@ -15,6 +15,7 @@ + diff --git a/src/components/ViewAdmin.vue b/src/components/ViewAdmin.vue index 55895fbab..bc465d77e 100644 --- a/src/components/ViewAdmin.vue +++ b/src/components/ViewAdmin.vue @@ -21,13 +21,13 @@ {{ t('recognize', 'The systemtags app is currently disabled. Some features of this app will not work.') }} - + {{ t('recognize', 'Could not execute the Node.js binary. You may need to set the path to a working binary manually.') }} {{ t('recognize', 'Background Jobs are not executed via cron. Recognize requires background jobs to be executed via cron.') }} -