diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fe0c806829a..d05e218fe6f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,8 +3,14 @@ # https://docs.github.com/en/free-pro-team@latest/actions/guides/building-and-testing-nodejs name: Build -# Run this Build for all pushes / PRs to current branch -on: [push, pull_request] +# Run this Build for pushes to our main and all PRs +on: + push: + branches: + - dtq-dev + - customer/* + pull_request: + workflow_dispatch: permissions: contents: read # to fetch code (actions/checkout) @@ -24,7 +30,6 @@ jobs: # Spin up UI on 127.0.0.1 to avoid host resolution issues in e2e tests with Node 20+ DSPACE_UI_HOST: 127.0.0.1 DSPACE_UI_PORT: 4000 - DSPACE_UI_BASEURL: http://127.0.0.1:4000 # Ensure all SSR caching is disabled in test environment DSPACE_CACHE_SERVERSIDE_BOTCACHE_MAX: 0 DSPACE_CACHE_SERVERSIDE_ANONYMOUSCACHE_MAX: 0 @@ -40,11 +45,8 @@ jobs: # Project name to use when running "docker compose" prior to e2e tests COMPOSE_PROJECT_NAME: 'ci' # Docker Registry to use for Docker compose scripts below. - # On the upstream dspace/dspace-angular repository we use GitHub's Container Registry - # (ghcr.io) to avoid aggressive rate limits at DockerHub. Forks cannot authenticate - # against ghcr.io/dspace/* with their own GITHUB_TOKEN (and the images there require - # auth), so on forks we fall back to docker.io where the same images are public. - DOCKER_REGISTRY: ${{ github.repository == 'dspace/dspace-angular' && 'ghcr.io' || 'docker.io' }} + # Using DockerHub as the images are publicly available there + DOCKER_REGISTRY: docker.io strategy: # Create a matrix of Node versions to test against (in parallel) matrix: @@ -124,18 +126,14 @@ jobs: path: 'coverage/dspace-angular/lcov.info' retention-days: 14 - # Login to our Docker registry, so that we can access Docker images using "docker compose" below. - # This login is required on the upstream repository because DOCKER_REGISTRY is set to ghcr.io - # and pulling ghcr.io/dspace/* requires authentication. On forks, DOCKER_REGISTRY falls back to - # docker.io (see env block above) where the same images are publicly pullable without a login, - # and forks cannot authenticate against ghcr.io/dspace/* with their own GITHUB_TOKEN anyway. - - name: Login to ${{ env.DOCKER_REGISTRY }} - uses: docker/login-action@v4 - if: github.repository == 'dspace/dspace-angular' - with: - registry: ${{ env.DOCKER_REGISTRY }} - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} + # Note: Docker login not required for public images from docker.io + # If you need to use ghcr.io images, uncomment the login step below: + # - name: Login to ${{ env.DOCKER_REGISTRY }} + # uses: docker/login-action@v3 + # with: + # registry: ${{ env.DOCKER_REGISTRY }} + # username: ${{ github.repository_owner }} + # password: ${{ secrets.GITHUB_TOKEN }} # Using "docker compose" start backend using CI configuration # and load assetstore from a cached copy @@ -315,40 +313,3 @@ jobs: - name: Shutdown Docker containers run: docker compose -f ./docker/docker-compose-ci.yml down - - # Codecov upload is a separate job in order to allow us to restart this separate from the entire build/test - # job above. This is necessary because Codecov uploads seem to randomly fail at times. - # See https://community.codecov.com/t/upload-issues-unable-to-locate-build-via-github-actions-api/3954 - codecov: - # Must run after 'tests' job above - needs: tests - # Only run on the upstream repository: forks do not have the CODECOV_TOKEN secret, - # and Codecov refuses to create a commit on a protected branch without a token. - if: github.repository == 'dspace/dspace-angular' - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v6 - - # Download artifacts from previous 'tests' job - - name: Download coverage artifacts - uses: actions/download-artifact@v8 - - # Now attempt upload to Codecov using its action. - # NOTE: We use a retry action to retry the Codecov upload if it fails the first time. - # - # Retry action: https://github.com/marketplace/actions/retry-action - # Codecov action: https://github.com/codecov/codecov-action - - name: Upload coverage to Codecov.io - uses: Wandalen/wretry.action@v3.8.0 - with: - action: codecov/codecov-action@v4 - # Ensure codecov-action throws an error when it fails to upload - # This allows us to auto-restart the action if an error is thrown - with: | - fail_ci_if_error: true - token: ${{ secrets.CODECOV_TOKEN }} - # Try re-running action 5 times max - attempt_limit: 5 - # Run again in 30 seconds - attempt_delay: 30000 diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 00000000000..6f6012db84f --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,160 @@ +# DSpace Docker deploy on dataquest servers +name: Deploy DSpace + +on: + workflow_call: + inputs: + DEV_MACHINE: + required: false + type: string + default: 'dev-6' + INSTANCE: + required: false + type: string + default: '8593' + IMPORT: + required: false + default: false + type: boolean + ERASE_DB: + required: false + default: false + type: boolean + + workflow_dispatch: + inputs: + INSTANCE: + required: true + default: '8593' + type: choice + options: + - '8593' + IMPORT: + required: true + default: true + type: boolean + ERASE_DB: + required: false + default: false + type: boolean + DEV_MACHINE: + required: false + type: string + default: 'dev-6' + +jobs: + deploy-3: + if: inputs.INSTANCE == '*' || inputs.INSTANCE == '8593' + runs-on: dspace-${{inputs.DEV_MACHINE}}-dep-1 + timeout-minutes: 60 + env: + INSTANCE: '8593' + CONFIG_PATH: /opt/dspace-envs/${{inputs.INSTANCE}} + ENVFILE: /opt/dspace-envs/${{inputs.INSTANCE}}/.env + steps: + - uses: actions/checkout@v6 + + - uses: ./.github/actions/erase-db + if: inputs.ERASE_DB + with: + INSTANCE: ${{ env.INSTANCE }} + NAME: dspace-${{ env.INSTANCE }} + + - name: Set start.sh as executable + run: chmod +x build-scripts/run/start.sh + + - name: deploy to ${{inputs.DEV_MACHINE}} + working-directory: build-scripts/run/ + env: + ADMIN_PASSWORD: ${{ secrets.DSPACE_ADMIN_PASSWORD }} + USER_PASSWORD: ${{ secrets.DSPACE_USER_PASSWORD }} + run: | + ./start.sh dspace-$INSTANCE + cd ../.. + docker compose --env-file $ENVFILE -p dspace-$INSTANCE -f docker/docker-compose.yml -f docker/docker-compose-rest.yml -f $CONFIG_PATH/docker-compose-rest.yml -f $CONFIG_PATH/docker-compose.yml up -d --no-build --remove-orphans + + playwright-after-deploy3: + needs: deploy-3 + if: '!inputs.IMPORT' + uses: ./.github/workflows/playwright-tests.yml + secrets: inherit + + playwright-after-import3: + needs: import-3 + if: inputs.IMPORT + uses: ./.github/workflows/playwright-tests.yml + secrets: inherit + + import-3: + runs-on: dspace-${{inputs.DEV_MACHINE}}-dep-1 + if: inputs.IMPORT + timeout-minutes: 900 + needs: deploy-3 + env: + INSTANCE: ${{inputs.INSTANCE}} + CONFIG_PATH: /opt/dspace-envs/ + FNAME: old_dump.sql + ADMIN_PASSWORD: ${{ secrets.ADMIN_PASSWORD }} + steps: + - name: vanilla import + run: | + export DATADIR=$CONFIG_PATH/${{inputs.INSTANCE}}/dump/$FNAME + export DNAME=dspace${{inputs.INSTANCE}} + export DDBNAME=dspacedb${{inputs.INSTANCE}} + + docker stop $DNAME || true + + echo "=====" + # echo Starting vanilla import DB + docker cp $DATADIR $DDBNAME:/tmp/$FNAME + + echo "Copying script for setup db roles to container..." + + # Create database, import dumpand extensions, then cleanup + docker exec $DDBNAME /bin/bash -c " + dropdb -U dspace -p 10593 dspace || true && + createdb -U dspace -p 10593 --owner=dspace --encoding=UNICODE dspace && + psql -p 10593 -U dspace -d dspace -f /tmp/$FNAME + " + echo "=====" + docker start $DNAME + + - name: run database migration + run: | + export DNAME=dspace${{inputs.INSTANCE}} + export DDBNAME=dspacedb${{inputs.INSTANCE}} + export SYNCSEQSETUP=$CONFIG_PATH/${{inputs.INSTANCE}}/$SYNCSEQFNAME + + echo "Starting DSpace container for migration..." + docker start $DNAME + + echo "Waiting longer for DSpace to be fully ready and locks to clear..." + sleep 20 + + echo "Running DSpace database migration..." + docker exec $DNAME /bin/bash -c "cd /dspace/bin && ./dspace database migrate ignored" + + echo "Waiting for DSpace to be ready..." + sleep 10 + + - name: restart dspace container + run: | + export DNAME=dspace${{inputs.INSTANCE}} + echo "Restarting DSpace container..." + docker restart $DNAME + sleep 60 + + - name: create administrator + run: | + export DNAME=dspace${{inputs.INSTANCE}} + echo "Creating DSpace administrator..." + docker exec $DNAME /bin/bash -c "cd /dspace/bin && ./dspace create-administrator -e dspace.admin.dev@dataquest.sk -f admin -l user -p \"${ADMIN_PASSWORD}\" -c en -o dataquest" + + - name: rebuild discovery index and run oai import + run: | + export DNAME=dspace${{inputs.INSTANCE}} + echo "Rebuilding discovery index..." + docker exec $DNAME /bin/bash -c "/dspace/bin/dspace index-discovery -b" + + echo "Running OAI import..." + docker exec $DNAME /bin/bash -c "/dspace/bin/dspace oai import -c" \ No newline at end of file diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 82b5d792502..fc367a69ea5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -5,15 +5,14 @@ name: Docker images # Also run for PRs to ensure PR doesn't break Docker build process # NOTE: uses "reusable-docker-build.yml" in DSpace/DSpace to actually build each of the Docker images # https://github.com/DSpace/DSpace/blob/main/.github/workflows/reusable-docker-build.yml -# +# on: push: branches: - - main - - 'dspace-**' - tags: - - 'dspace-**' + - dtq-dev + - customer/* pull_request: + workflow_dispatch: permissions: contents: read # to fetch code (actions/checkout) @@ -25,32 +24,36 @@ jobs: ############################################################# dspace-angular: # Ensure this job never runs on forked repos. It's only executed for 'dspace/dspace-angular' - if: github.repository == 'dspace/dspace-angular' + if: github.repository == 'dataquest-dev/dspace-angular' # Use the reusable-docker-build.yml script from DSpace/DSpace repo to build our Docker image - uses: DSpace/DSpace/.github/workflows/reusable-docker-build.yml@main + uses: dataquest-dev/DSpace/.github/workflows/reusable-docker-build.yml@dtq-dev with: build_id: dspace-angular-dev - image_name: dspace/dspace-angular + image_name: dataquest/dspace-angular dockerfile_path: ./Dockerfile + tags_flavor: suffix=-dev + # As this is a "dev" image, its tags are all suffixed with "-dev". Otherwise, it uses the same + # tagging logic as the primary 'dspace/dspace-angular' image above. + # run_python_version_script: true + # python_version_script_dest: src/static-files/VERSION_D.html secrets: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }} ############################################################# - # Build/Push the 'dspace/dspace-angular' image ('-dist' tag) + # Build/Push the 'dataquest/dspace-angular' image ('-dist' tag) ############################################################# dspace-angular-dist: - # Ensure this job never runs on forked repos. It's only executed for 'dspace/dspace-angular' - if: github.repository == 'dspace/dspace-angular' - # Use the reusable-docker-build.yml script from DSpace/DSpace repo to build our Docker image - uses: DSpace/DSpace/.github/workflows/reusable-docker-build.yml@main + # Ensure this job never runs on forked repos. It's only executed for 'dataquest/dspace-angular' + if: github.repository == 'dataquest-dev/dspace-angular' + # Use the reusable-docker-build.yml script from dataquest-dev/DSpace repo to build our Docker image + uses: dataquest-dev/DSpace/.github/workflows/reusable-docker-build.yml@dtq-dev with: - build_id: dspace-angular-dist - image_name: dspace/dspace-angular + build_id: dspace-angular + image_name: dataquest/dspace-angular dockerfile_path: ./Dockerfile.dist - # As this is a "dist" image, its tags are all suffixed with "-dist". Otherwise, it uses the same - # tagging logic as the primary 'dspace/dspace-angular' image above. - tags_flavor: suffix=-dist + # run_python_version_script: true + # python_version_script_dest: src/static-files/VERSION_D.html secrets: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }} @@ -59,121 +62,13 @@ jobs: REDEPLOY_SANDBOX_URL: ${{ secrets.REDEPLOY_SANDBOX_URL }} REDEPLOY_DEMO_URL: ${{ secrets.REDEPLOY_DEMO_URL }} - ################################################################################# - # Test Deployment via Docker to ensure newly built images are working properly - ################################################################################# - docker-deploy: - # Ensure this job never runs on forked repos. It's only executed for 'dspace/dspace-angular' - if: github.repository == 'dspace/dspace-angular' - runs-on: ubuntu-latest - # Must run after all major images are built - needs: [dspace-angular, dspace-angular-dist] - env: - # Override default dspace.server.url & REST 'host' because backend starts at http://127.0.0.1:8080 - dspace__P__server__P__url: http://127.0.0.1:8080/server - DSPACE_REST_HOST: 127.0.0.1 - # Override default dspace.ui.url to also use 127.0.0.1. - dspace__P__ui__P__url: http://127.0.0.1:4000 - # Override default ui.baseUrl to also use 127.0.0.1. This must match 'dspace.ui.url' for SSR to be enabled. - DSPACE_UI_BASEURL: http://127.0.0.1:4000 - steps: - # Checkout our codebase (to get access to Docker Compose scripts) - - name: Checkout codebase - uses: actions/checkout@v6 - # Download Docker image artifacts (which were just built by reusable-docker-build.yml) - - name: Download Docker image artifacts - uses: actions/download-artifact@v8 - with: - # Download all amd64 Docker images (TAR files) into the /tmp/docker directory - pattern: docker-image-*-linux-amd64 - path: /tmp/docker - merge-multiple: true - # Load each of the images into Docker by calling "docker image load" for each. - # This ensures we are using the images just built & not any prior versions on DockerHub - - name: Load all downloaded Docker images - run: | - find /tmp/docker -type f -name "*.tar" -exec docker image load --input "{}" \; - docker image ls -a - # Start backend using our compose script in the codebase. - - name: Start backend in Docker - # MUST use docker.io as we don't have a copy of this backend image in our GitHub Action, - # and docker.io is the only public image. If we ever hit aggressive rate limits at DockerHub, - # we may need to consider making the 'ghcr.io' images public & switch this to 'ghcr.io' - env: - DOCKER_REGISTRY: docker.io - run: | - docker compose -f docker/docker-compose-rest.yml up -d - sleep 10 - docker container ls - # Create a test admin account. Load test data from a simple set of AIPs as defined in cli.ingest.yml - # NOTE: Before creating test data, we wait for the backend to become responsive by requesting it every 10 sec. - # Timeout after 5 minutes. This is done to ensure the backend is fully initialized before we create test data. - - name: Load test data into Backend - run: | - timeout 5m wget --retry-connrefused -t 0 --waitretry=10 http://127.0.0.1:8080/server/api - docker compose -f docker/cli.yml run --rm dspace-cli create-administrator -e test@test.edu -f admin -l user -p admin -c en - docker compose -f docker/cli.yml -f docker/cli.ingest.yml run --rm dspace-cli - # Verify backend started successfully. - # 1. Make sure root endpoint is responding (check for dspace.name defined in docker-compose.yml) - # 2. Also check /collections endpoint to ensure the test data loaded properly (check for a collection name in AIPs) - - name: Verify backend is responding properly - run: | - result=$(wget -O- -q http://127.0.0.1:8080/server/api) - echo "$result" - echo "$result" | grep -oE "\"DSpace Started with Docker Compose\"" - result=$(wget -O- -q http://127.0.0.1:8080/server/api/core/collections) - echo "$result" - echo "$result" | grep -oE "\"Dog in Yard\"" - # Start production frontend using our compose script in the codebase. - - name: Start production frontend in Docker - # Specify the GHCR copy of the production frontend, so that we use the newly built image - env: - DOCKER_REGISTRY: ghcr.io - run: | - docker compose -f docker/docker-compose-dist.yml up -d - sleep 10 - docker container ls - # Verify production frontend started successfully. - # 1. Make sure /home path has "DSpace software" (this is in the footer of the page) - # 2. Also check /community-list page lists one of the test Communities in the loaded test data - - name: Verify production frontend is responding properly - run: | - result=$(wget -O- -q http://127.0.0.1:4000/home) - echo "$result" - echo "$result" | grep -oE "\"DSpace software\"" - - name: Error logs of production frontend (if error in startup) - if: ${{ failure() }} - run: | - docker compose -f docker/docker-compose-dist.yml logs - # Now shutdown the production frontend image and startup the development frontend image - - name: Shutdown production frontend - run: | - docker compose -f docker/docker-compose-dist.yml down - sleep 10 - docker container ls - - name: Startup development frontend - # Specify the GHCR copy of the development frontend, so that we use the newly built image - env: - DOCKER_REGISTRY: ghcr.io - run: | - docker compose -f docker/docker-compose.yml up -d - sleep 10 - docker container ls - # Verify development frontend started successfully. - # 1. First, keep requesting the frontend every 10 seconds to wait until its up. Timeout after 10 minutes. - # 2. Once it's responding, check to see if the word "DSpace" appears. - # We cannot check for anything more specific because development mode doesn't have SSR. - - name: Verify development frontend is responding properly - run: | - timeout 10m wget --retry-connrefused -t 0 --waitretry=10 http://127.0.0.1:4000 - result=$(wget -O- -q http://127.0.0.1:4000) - echo "$result" - echo "$result" | grep -oE "DSpace" - - name: Error logs of development frontend (if error in startup) - if: ${{ failure() }} - run: | - docker compose -f docker/docker-compose.yml logs - # Shutdown our containers - - name: Shutdown running Docker containers - run: | - docker compose -f docker/docker-compose.yml -f docker/docker-compose-rest.yml down \ No newline at end of file + deploy: + needs: dspace-angular + uses: dataquest-dev/dspace-angular/.github/workflows/deploy.yml@customer/jcu + if: ${{ github.event_name != 'pull_request' }} + with: + INSTANCE: '8593' + IMPORT: false + ERASE_DB: false + DEV_MACHINE: 'dev-6' + secrets: inherit diff --git a/.github/workflows/playwright-tests.yml b/.github/workflows/playwright-tests.yml new file mode 100644 index 00000000000..fed8ac7d7aa --- /dev/null +++ b/.github/workflows/playwright-tests.yml @@ -0,0 +1,39 @@ +name: Playwright tests + +on: + workflow_dispatch: + workflow_call: + +jobs: + playwright-tests: + runs-on: dspace-test-1 + timeout-minutes: 45 + steps: + - name: Checkout Playwright tests + uses: actions/checkout@v6 + with: + repository: dataquest-dev/dspace-ui-tests + path: dspace-ui-tests + token: ${{ secrets.DEPLOY_DEV5_GH_ACTION_DISPATCH }} + + - name: Checkout main repository + uses: actions/checkout@v6 + with: + path: this-repository + + - name: Copy customer config to UI tests repository for merging + run: | + cp this-repository/config/config.jcu.ui.tests.json dspace-ui-tests/customer-constants/ + + - name: test + working-directory: dspace-ui-tests/scripts + env: + HOME_URL: https://dev-6.pc:8593/ + NAME: JCU + run: | + ./test.sh + + - uses: mazoea/ga-maz/end@master + with: + CHOWN_DIR: ./ + if: ${{ always() }} \ No newline at end of file diff --git a/build-scripts/run/start.sh b/build-scripts/run/start.sh new file mode 100644 index 00000000000..99461633407 --- /dev/null +++ b/build-scripts/run/start.sh @@ -0,0 +1,61 @@ +#!/bin/bash +if [[ "x$ENVFILE" == "x" ]]; then + export ENVFILE=$(pwd)/envs/.default + echo "Using default envfile" +fi + +PROJECT=${1:-unnamed_dspace} + +echo "Using envfile: [$ENVFILE] for project: [$PROJECT]" + +source $ENVFILE + +# docker-compose does not pull those that have `build` section?! +echo "=====" +docker pull $DSPACE_UI_IMAGE + +pushd ../.. +echo "=====" +docker compose --env-file $ENVFILE -f docker/docker-compose.yml -f docker/docker-compose-rest.yml pull +docker compose --env-file $ENVFILE -p $PROJECT -f docker/docker-compose.yml -f docker/docker-compose-rest.yml up -d --no-build --remove-orphans +popd + +# Create admin user +# set DOCKER_OWNER to match our image (see cli.yml) +pushd ../.. +echo "=====" +#docker compose --env-file $ENVFILE -p $PROJECT -f docker/matomo-w-db.yml pull +#docker compose --env-file $ENVFILE -p $PROJECT -f docker/matomo-w-db.yml up -d --no-build + +# Ensure CLI container uses the same project network +export COMPOSE_PROJECT_NAME=$PROJECT + +# docker-compose-rest.yml must be last, since it specifies network in more detail. If it is not last, there is "root must be a mapping" error. +if [[ -z "$ADMIN_PASSWORD" ]]; then + echo "ERROR: ADMIN_PASSWORD is required but not set." >&2 + exit 1 +fi +if [[ -z "$USER_PASSWORD" ]]; then + echo "ERROR: USER_PASSWORD is required but not set." >&2 + exit 1 +fi +echo "Creating administrator user..." +docker compose --env-file $ENVFILE -p $PROJECT -f docker/docker-compose.yml -f docker/cli.yml -f docker/docker-compose-rest.yml run --rm dspace-cli create-administrator -e dspace.admin.dev@dataquest.sk -f admin -l user -p "${ADMIN_PASSWORD}" -c en + +echo "Creating regular user..." +docker compose --env-file $ENVFILE -p $PROJECT -f docker/docker-compose.yml -f docker/cli.yml -f docker/docker-compose-rest.yml run --rm dspace-cli user --add -m dspace.user.dev@dataquest.sk -g meno -s priezvisko -l en -p "${USER_PASSWORD}" + +echo "Checking DSpace version..." +docker compose --env-file $ENVFILE -p $PROJECT -f docker/docker-compose.yml -f docker/cli.yml -f docker/docker-compose-rest.yml run --rm dspace-cli version + +echo "=====" +echo "Logs" +docker compose --env-file $ENVFILE -p $PROJECT -f docker/docker-compose.yml -f docker/docker-compose-rest.yml logs -n 50 || true +popd + +echo "=====" +echo "Copy assetstore" +docker cp assetstore dspace${INSTANCE}:/dspace/ + +echo "=====" +echo "Finished start.sh" diff --git a/config/config.jcu.ui.tests.json b/config/config.jcu.ui.tests.json new file mode 100644 index 00000000000..d716cedbc72 --- /dev/null +++ b/config/config.jcu.ui.tests.json @@ -0,0 +1,16 @@ +{ + "env": { + "has_discojuice": false, + "has_whats_new": false, + "has_profile_and_logout_link": true, + "has_static_page": false + }, + "locators": { + "title": "DSpace Repository :: Home", + "profile_link": "a[href=\"/profile\"]", + "logout_link": "button[data-test=\"logout-button\"]", + "signon_link": "button[data-test=\"login-menu\"]", + "result_card": "li[data-test=\"list-object\"]", + "new_section": "nav#admin-sidebar" + } +} \ No newline at end of file