diff --git a/.dockerignore b/.dockerignore index 623b3a1b65da..7d3bdc2b4b0d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,6 +4,7 @@ */target/ dspace/modules/*/target/ Dockerfile.* -dspace/src/main/docker/dspace-postgres-loadsql +dspace/src/main/docker/dspace-postgres-pgcrypto +dspace/src/main/docker/dspace-postgres-pgcrypto-curl dspace/src/main/docker/README.md dspace/src/main/docker-compose/ diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ea8cb99b44de..8b522c1a2dc5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,8 +3,14 @@ # https://docs.github.com/en/free-pro-team@latest/actions/guides/building-and-testing-java-with-maven name: Build -# Run this Build for all pushes / PRs to current branch -on: [push, pull_request] +# Run this Build for pushes to our main and all PRs +on: + push: + branches: + - dtq-dev + - customer/* + pull_request: + workflow_dispatch: permissions: contents: read # to fetch code (actions/checkout) @@ -81,9 +87,6 @@ jobs: # Codecov upload is a separate job in order to allow us to restart this separate from the entire build/test # job above. This is necessary because Codecov uploads seem to randomly fail at times. # See https://community.codecov.com/t/upload-issues-unable-to-locate-build-via-github-actions-api/3954 - # Codecov upload disabled for the JCU fork: the repository is not configured with a - # CODECOV_TOKEN and the project-level coverage check fails on customization commits. - # Coverage is still produced and uploaded as a build artifact by the 'tests' job above. # codecov: # # Must run after 'tests' job above # needs: tests diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1e83dd18e1a3..d2b558dd0a41 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,52 +1,54 @@ # DSpace Docker image build for hub.docker.com name: Docker images -# Run this Build for all pushes to 'main' or maintenance branches, or tagged releases. +# Run this Build for all pushes to dtq-dev branch # Also run for PRs to ensure PR doesn't break Docker build process # NOTE: uses "reusable-docker-build.yml" to actually build each of the Docker images. on: push: branches: - - main - - 'dspace-**' - tags: - - 'dspace-**' + - dtq-dev + - customer/* + pull_request: + workflow_dispatch: permissions: contents: read # to fetch code (actions/checkout) - packages: write # to write images to GitHub Container Registry (GHCR) jobs: #################################################### - # Build/Push the 'dspace/dspace-dependencies' image. + # Build/Push the 'dataquest/dspace-dependencies' image. # This image is used by all other DSpace build jobs. #################################################### dspace-dependencies: - # Ensure this job never runs on forked repos. It's only executed for 'dspace/dspace' - if: github.repository == 'dspace/dspace' + # Ensure this job never runs on forked repos. It's only executed for 'dataquest/dspace' + if: github.repository == 'dataquest-dev/dspace' + secrets: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }} uses: ./.github/workflows/reusable-docker-build.yml with: build_id: dspace-dependencies - image_name: dspace/dspace-dependencies + image_name: dataquest/dspace-dependencies dockerfile_path: ./Dockerfile.dependencies - secrets: - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }} + ####################################### - # Build/Push the 'dspace/dspace' image + # Build/Push the 'dataquest/dspace' image ####################################### dspace: - # Ensure this job never runs on forked repos. It's only executed for 'dspace/dspace' - if: github.repository == 'dspace/dspace' + # Ensure this job never runs on forked repos. It's only executed for 'dataquest/dspace' + if: github.repository == 'dataquest-dev/dspace' # Must run after 'dspace-dependencies' job above needs: dspace-dependencies uses: ./.github/workflows/reusable-docker-build.yml with: build_id: dspace-prod - image_name: dspace/dspace + image_name: dataquest/dspace dockerfile_path: ./Dockerfile + run_python_version_script: true + python_version_script_dest: dspace/config/VERSION_D.txt secrets: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }} @@ -56,52 +58,52 @@ jobs: REDEPLOY_DEMO_URL: ${{ secrets.REDEPLOY_DEMO_URL }} ############################################################# - # Build/Push the 'dspace/dspace' image ('-test' tag) + # Build/Push the 'dataquest/dspace' image ('-test' tag) ############################################################# dspace-test: - # Ensure this job never runs on forked repos. It's only executed for 'dspace/dspace' - if: github.repository == 'dspace/dspace' + # Ensure this job never runs on forked repos. It's only executed for 'dataquest/dspace' + if: github.repository == 'dataquest-dev/dspace' # Must run after 'dspace-dependencies' job above needs: dspace-dependencies uses: ./.github/workflows/reusable-docker-build.yml with: build_id: dspace-test - image_name: dspace/dspace + image_name: dataquest/dspace dockerfile_path: ./Dockerfile.test # As this is a test/development image, its tags are all suffixed with "-test". Otherwise, it uses the same - # tagging logic as the primary 'dspace/dspace' image above. + # tagging logic as the primary 'dataquest/dspace' image above. tags_flavor: suffix=-test secrets: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }} ########################################### - # Build/Push the 'dspace/dspace-cli' image + # Build/Push the 'dataquest/dspace-cli' image ########################################### dspace-cli: - # Ensure this job never runs on forked repos. It's only executed for 'dspace/dspace' - if: github.repository == 'dspace/dspace' + # Ensure this job never runs on forked repos. It's only executed for 'dataquest/dspace' + if: github.repository == 'dataquest-dev/dspace' # Must run after 'dspace-dependencies' job above needs: dspace-dependencies uses: ./.github/workflows/reusable-docker-build.yml with: build_id: dspace-cli - image_name: dspace/dspace-cli + image_name: dataquest/dspace-cli dockerfile_path: ./Dockerfile.cli secrets: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }} ########################################### - # Build/Push the 'dspace/dspace-solr' image + # Build/Push the 'dataquest/dspace-solr' image ########################################### dspace-solr: - # Ensure this job never runs on forked repos. It's only executed for 'dspace/dspace' - if: github.repository == 'dspace/dspace' + # Ensure this job never runs on forked repos. It's only executed for 'dataquest/dspace' + if: github.repository == 'dataquest-dev/dspace' uses: ./.github/workflows/reusable-docker-build.yml with: build_id: dspace-solr - image_name: dspace/dspace-solr + image_name: dataquest/dspace-solr dockerfile_path: ./dspace/src/main/docker/dspace-solr/Dockerfile # Must pass solrconfigs to the Dockerfile so that it can find the required Solr config files dockerfile_additional_contexts: 'solrconfigs=./dspace/solr/' @@ -113,133 +115,55 @@ jobs: REDEPLOY_SANDBOX_URL: ${{ secrets.REDEPLOY_SANDBOX_SOLR_URL }} REDEPLOY_DEMO_URL: ${{ secrets.REDEPLOY_DEMO_SOLR_URL }} - ######################################################## - # Build/Push the 'dspace/dspace-postgres-loadsql' image - ######################################################## - dspace-postgres-loadsql: - # Ensure this job never runs on forked repos. It's only executed for 'dspace/dspace' - if: github.repository == 'dspace/dspace' + ########################################################### + # Build/Push the 'dataquest/dspace-postgres-pgcrypto' image + ########################################################### + dspace-postgres-pgcrypto: + # Ensure this job never runs on forked repos. It's only executed for 'dataquest/dspace' + if: github.repository == 'dataquest-dev/dspace' + uses: ./.github/workflows/reusable-docker-build.yml + with: + build_id: dspace-postgres-pgcrypto + image_name: dataquest/dspace-postgres-pgcrypto + # Must build out of subdirectory to have access to install script for pgcrypto. + # NOTE: this context will build the image based on the Dockerfile in the specified directory + dockerfile_context: ./dspace/src/main/docker/dspace-postgres-pgcrypto/ + secrets: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }} + + ######################################################################## + # Build/Push the 'dataquest/dspace-postgres-pgcrypto' image (-loadsql tag) + ######################################################################## + dspace-postgres-pgcrypto-loadsql: + # Ensure this job never runs on forked repos. It's only executed for 'dataquest/dspace' + if: github.repository == 'dataquest-dev/dspace' uses: ./.github/workflows/reusable-docker-build.yml with: - build_id: dspace-postgres-loadsql - image_name: dspace/dspace-postgres-loadsql - # Must build out of subdirectory to have access to install script. + build_id: dspace-postgres-pgcrypto-loadsql + image_name: dataquest/dspace-postgres-pgcrypto + # Must build out of subdirectory to have access to install script for pgcrypto. # NOTE: this context will build the image based on the Dockerfile in the specified directory - dockerfile_context: ./dspace/src/main/docker/dspace-postgres-loadsql/ + dockerfile_context: ./dspace/src/main/docker/dspace-postgres-pgcrypto-curl/ + # Suffix all tags with "-loadsql". Otherwise, it uses the same + # tagging logic as the primary 'dataquest/dspace-postgres-pgcrypto' image above. + tags_flavor: suffix=-loadsql secrets: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }} - ################################################################################# - # Test Deployment via Docker to ensure newly built images are working properly - ################################################################################# - docker-deploy: - # Ensure this job never runs on forked repos. It's only executed for 'dspace/dspace' - if: github.repository == 'dspace/dspace' + + dspace-redeploy: runs-on: ubuntu-latest - # Must run after all major images are built - needs: [dspace, dspace-test, dspace-cli, dspace-solr] - env: - # Override defaults dspace.server.url because backend starts at http://127.0.0.1:8080 - dspace__P__server__P__url: http://127.0.0.1:8080/server - # Enable all optional modules / controllers for this test deployment. - # This helps check for errors in deploying these modules via Spring Boot - iiif__P__enabled: true - ldn__P__enabled: true - oai__P__enabled: true - rdf__P__enabled: true - signposting__P__enabled: true - sword__D__server__P__enabled: true - swordv2__D__server__P__enabled: true - # If this is a PR against main (default branch), use "latest". - # Else if this is a PR against a different branch, used the base branch name. - # Else if this is a commit on main (default branch), use the "latest" tag. - # Else, just use the branch name. - # NOTE: DSPACE_VER is used because our docker compose scripts default to using the "-test" image. - DSPACE_VER: ${{ (github.event_name == 'pull_request' && github.event.pull_request.base.ref == github.event.repository.default_branch && 'latest') || (github.event_name == 'pull_request' && github.event.pull_request.base.ref) || (github.ref_name == github.event.repository.default_branch && 'latest') || github.ref_name }} - # Docker Registry to use for Docker compose scripts below. - # We use GitHub's Container Registry to avoid aggressive rate limits at DockerHub. - DOCKER_REGISTRY: ghcr.io + needs: dspace + if: false steps: - # Checkout our codebase (to get access to Docker Compose scripts) - - name: Checkout codebase - uses: actions/checkout@v6 - # Download Docker image artifacts (which were just built by reusable-docker-build.yml) - - name: Download Docker image artifacts - uses: actions/download-artifact@v8 - with: - # Download all amd64 Docker images (TAR files) into the /tmp/docker directory - pattern: docker-image-*-linux-amd64 - path: /tmp/docker - merge-multiple: true - # Load each of the images into Docker by calling "docker image load" for each. - # This ensures we are using the images just built & not any prior versions on DockerHub - - name: Load all downloaded Docker images - run: | - find /tmp/docker -type f -name "*.tar" -exec docker image load --input "{}" \; - docker image ls -a - # Start backend using our compose script in the codebase. - - name: Start backend in Docker - run: | - docker compose -f docker-compose.yml up -d - sleep 10 - docker container ls - # Create a test admin account. Load test data from a simple set of AIPs as defined in cli.ingest.yml - # NOTE: Before creating test data, we wait for the backend to become responsive by requesting it every 10 sec. - # Timeout after 5 minutes. This is done to ensure the backend is fully initialized before we create test data. - - name: Load test data into Backend - run: | - timeout 5m wget --retry-connrefused -t 0 --waitretry=10 http://127.0.0.1:8080/server/api - docker compose -f docker-compose-cli.yml run --rm dspace-cli create-administrator -e test@test.edu -f admin -l user -p admin -c en - docker compose -f docker-compose-cli.yml -f dspace/src/main/docker-compose/cli.ingest.yml run --rm dspace-cli - # Verify backend started successfully. - # 1. Make sure root endpoint is responding (check for dspace.name defined in docker-compose.yml) - # 2. Also check /collections endpoint to ensure the test data loaded properly (check for a collection name in AIPs) - - name: Verify backend is responding properly - run: | - result=$(wget -O- -q http://127.0.0.1:8080/server/api) - echo "$result" - echo "$result" | grep -oE "\"DSpace Started with Docker Compose\"," - result=$(wget -O- -q http://127.0.0.1:8080/server/api/core/collections) - echo "$result" - echo "$result" | grep -oE "\"Dog in Yard\"," - # Verify basic backend logging is working. - # 1. Access the top communities list. Verify that the "Before request" INFO statement is logged - # 2. Access an invalid endpoint (and ignore 404 response). Verify that a "status:404" WARN statement is logged - - name: Verify backend is logging properly - run: | - wget -O/dev/null -q http://127.0.0.1:8080/server/api/core/communities/search/top - logs=$(docker compose -f docker-compose.yml logs -n 5 dspace) - echo "$logs" - echo "$logs" | grep -o "Before request \[GET /server/api/core/communities/search/top\]" - wget -O/dev/null -q http://127.0.0.1:8080/server/api/does/not/exist || true - logs=$(docker compose -f docker-compose.yml logs -n 5 dspace) - echo "$logs" - echo "$logs" | grep -o "status:404 exception: The repository type does.not was not found" - # Verify Handle Server can be stared and is working properly - # 1. First generate the "[dspace]/handle-server" folder with the sitebndl.zip - # 2. Start the Handle Server (and wait 20 seconds to let it start up) - # 3. Verify logs do NOT include "Exception" in the text (as that means an error occurred) - # 4. Check that Handle Proxy HTML page is responding on default port (8000) - - name: Verify Handle Server is working properly - run: | - docker exec -i dspace /dspace/bin/make-handle-config - echo "Starting Handle Server..." - docker exec -i dspace /dspace/bin/start-handle-server - sleep 20 - echo "Checking for errors in error.log" - result=$(docker exec -i dspace sh -c "cat /dspace/handle-server/logs/error.log* || echo ''") - echo "$result" - echo "$result" | grep -vqz "Exception" - echo "Checking for errors in handle-server.log..." - result=$(docker exec -i dspace cat /dspace/log/handle-server.log) - echo "$result" - echo "$result" | grep -vqz "Exception" - echo "Checking to see if Handle Proxy webpage is available..." - result=$(wget -O- -q http://127.0.0.1:8000/) - echo "$result" - echo "$result" | grep -oE "Handle Proxy" - # Shutdown our containers - - name: Shutdown Docker containers + - name: redeploy + if: '!cancelled()' run: | - docker compose -f docker-compose.yml down + curl -H "Accept: application/vnd.github.everest-preview+json" \ + -H "Authorization: token ${{ secrets.DEPLOY_DEV5_GH_ACTION_DISPATCH }}" \ + --request POST \ + https://api.github.com/repos/dataquest-dev/\ + dspace-angular/actions/workflows/deploy.yml/dispatches \ + --data "{\"ref\":\"refs/heads/dtq-dev\"}" diff --git a/.github/workflows/reusable-docker-build.yml b/.github/workflows/reusable-docker-build.yml index eda8ec294b1d..44ae5e3f8bfe 100644 --- a/.github/workflows/reusable-docker-build.yml +++ b/.github/workflows/reusable-docker-build.yml @@ -8,6 +8,14 @@ on: workflow_call: # Possible Inputs to this reusable job inputs: + python_version_script_dest: + required: false + default: version.txt + type: string + run_python_version_script: + required: false + default: false + type: boolean # Build name/id for this Docker build. Used for digest storage to avoid digest overlap between builds. build_id: required: true @@ -58,9 +66,9 @@ env: # e.g. PR against 'main' will use "latest". a PR against 'dspace-7_x' will use 'dspace-7_x'. IMAGE_TAGS: | type=raw,value=latest,enable=${{ github.ref_name == github.event.repository.default_branch }} + type=raw,value=${{ github.sha }} type=ref,event=branch,enable=${{ github.ref_name != github.event.repository.default_branch }} type=ref,event=tag - type=raw,value=${{ (github.event.pull_request.base.ref == github.event.repository.default_branch && 'latest') || github.event.pull_request.base.ref }},enable=${{ github.event_name == 'pull_request' }} # Define default tag "flavor" for docker/metadata-action per # https://github.com/docker/metadata-action#flavor-input # We manage the 'latest' tag ourselves to the 'main' branch (see settings above) @@ -77,7 +85,7 @@ env: DEPLOY_ARCH: 'linux/amd64' # Registry used during building of Docker images. (All images are later copied to docker.io registry) # We use GitHub's Container Registry to avoid aggressive rate limits at DockerHub. - DOCKER_BUILD_REGISTRY: ghcr.io + DOCKER_BUILD_REGISTRY: docker.io jobs: docker-build: @@ -85,17 +93,19 @@ jobs: strategy: matrix: # Architectures / Platforms for which we will build Docker images - arch: [ 'linux/amd64', 'linux/arm64' ] + # arch: [ 'linux/amd64', 'linux/arm64' ] + arch: [ 'linux/amd64' ] + os: [ ubuntu-latest ] isPr: - ${{ github.event_name == 'pull_request' }} - # If this is a PR, we ONLY build for AMD64. For PRs we only do a sanity check test to ensure Docker builds work. - # The below exclude therefore ensures we do NOT build ARM64 for PRs. - exclude: - - isPr: true - arch: linux/arm64 + # If this is a PR, we ONLY build for AMD64. For PRs we only do a sanity check test to ensure Docker builds work. + # The below exclude therefore ensures we do NOT build ARM64 for PRs. + # exclude: + # - isPr: true + # os: ubuntu-latest + # arch: linux/arm64 - # If ARM64, then use the Ubuntu ARM64 runner. Otherwise, use the Ubuntu AMD64 runner - runs-on: ${{ matrix.arch == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} + runs-on: ${{ matrix.os }} steps: # This step converts the slashes in the "arch" matrix values above into dashes & saves to env.ARCH_NAME @@ -111,15 +121,21 @@ jobs: - name: Checkout codebase uses: actions/checkout@v6 + - name: Add version + if: ${{ inputs.run_python_version_script }} + run: python scripts/sourceversion.py ${{ github.server_url }}/${{ github.repository }}/actions/runs/ ${{ github.run_id }} > ${{ inputs.python_version_script_dest }} + + # https://github.com/docker/login-action # NOTE: This login occurs for BOTH non-PRs or PRs. PRs *must* also login to access private images from GHCR # during the build process - name: Login to ${{ env.DOCKER_BUILD_REGISTRY }} + # Only login if not a PR, as PRs only trigger a Docker build and not a push + if: ${{ ! matrix.isPr }} uses: docker/login-action@v4 with: - registry: ${{ env.DOCKER_BUILD_REGISTRY }} - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_ACCESS_TOKEN }} # https://github.com/docker/setup-buildx-action - name: Setup Docker Buildx @@ -164,193 +180,4 @@ jobs: # Use GitHub cache to load cached Docker images and cache the results of this build # This decreases the number of images we need to fetch from DockerHub cache-from: type=gha,scope=${{ inputs.build_id }} - cache-to: type=gha,scope=${{ inputs.build_id }},mode=min - - # Export the digest of Docker build locally - - name: Export Docker build digest - if: ${{ ! matrix.isPr }} - run: | - mkdir -p /tmp/digests - digest="${{ steps.docker_build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - # Upload digest to an artifact, so that it can be used in combined manifest below - # (The purpose of the combined manifest is to list both amd64 and arm64 builds under same tag) - - name: Upload Docker build digest to artifact - if: ${{ ! matrix.isPr }} - uses: actions/upload-artifact@v7 - with: - name: digests-${{ inputs.build_id }}-${{ env.ARCH_NAME }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 - - #------------------------------------------------------------------------------ - # Second, we build the image again in order to store it in a local TAR file. - # This TAR of the image is cached/saved as an artifact, so that it can be used - # by later jobs to install the brand-new images for automated testing. - # This TAR build is performed BOTH for PRs and for branch commits (non-PRs). - # - # (This approach has the advantage of avoiding having to download the newly built - # image from DockerHub or GHCR during automated testing.) - # - # See the 'docker-deploy' job in docker.yml as an example of where this TAR is used. - #------------------------------------------------------------------------------- - # Build local image (again) and store in a TAR file in /tmp directory - # This step is only done for AMD64, as that's the only image we use in our automated testing (at this time). - # NOTE: This step cannot be combined with the build above as it's a different type of output. - - name: Build and push image to local TAR file - if: ${{ matrix.arch == 'linux/amd64'}} - uses: docker/build-push-action@v7 - with: - build-contexts: | - ${{ inputs.dockerfile_additional_contexts }} - context: ${{ inputs.dockerfile_context }} - file: ${{ inputs.dockerfile_path }} - # Tell DSpace's Docker files to use the build registry instead of DockerHub - build-args: - DOCKER_REGISTRY=${{ env.DOCKER_BUILD_REGISTRY }} - platforms: ${{ matrix.arch }} - tags: ${{ steps.meta_build.outputs.tags }} - labels: ${{ steps.meta_build.outputs.labels }} - # Use GitHub cache to load cached Docker images and cache the results of this build - # This decreases the number of images we need to fetch from DockerHub - cache-from: type=gha,scope=${{ inputs.build_id }} - cache-to: type=gha,scope=${{ inputs.build_id }},mode=min - # Export image to a local TAR file - outputs: type=docker,dest=/tmp/${{ inputs.build_id }}.tar - - # Upload the local docker image (in TAR file) to a build Artifact - # This step is only done for AMD64, as that's the only image we use in our automated testing (at this time). - - name: Upload local image TAR to artifact - if: ${{ matrix.arch == 'linux/amd64'}} - uses: actions/upload-artifact@v7 - with: - name: docker-image-${{ inputs.build_id }}-${{ env.ARCH_NAME }} - path: /tmp/${{ inputs.build_id }}.tar - if-no-files-found: error - retention-days: 1 - - ########################################################################################## - # Merge Docker digests (from various architectures) into a single manifest. - # This runs after all Docker builds complete above. The purpose is to include all builds - # under a single manifest for this tag. - # (e.g. both linux/amd64 and linux/arm64 should be listed under the same tagged Docker image) - ########################################################################################## - docker-build_manifest: - # Only run if this is NOT a PR - if: ${{ github.event_name != 'pull_request' }} - runs-on: ubuntu-latest - needs: - - docker-build - steps: - - name: Download Docker build digests - uses: actions/download-artifact@v8 - with: - path: /tmp/digests - # Download digests for both AMD64 and ARM64 into same directory - pattern: digests-${{ inputs.build_id }}-* - merge-multiple: true - - - name: Login to ${{ env.DOCKER_BUILD_REGISTRY }} - uses: docker/login-action@v4 - with: - registry: ${{ env.DOCKER_BUILD_REGISTRY }} - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 - - - name: Add Docker metadata for image - id: meta - uses: docker/metadata-action@v6 - with: - images: ${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }} - tags: ${{ env.IMAGE_TAGS }} - flavor: ${{ env.TAGS_FLAVOR }} - - - name: Create manifest list from digests and push to ${{ env.DOCKER_BUILD_REGISTRY }} - working-directory: /tmp/digests - run: | - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *) - - - name: Inspect manifest in ${{ env.DOCKER_BUILD_REGISTRY }} - run: | - docker buildx imagetools inspect ${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }} - - ########################################################################################## - # Copy images / manifest to DockerHub. - # This MUST run after *both* images (AMD64 and ARM64) are built and uploaded to GitHub - # Container Registry (GHCR). Attempting to run this in parallel to GHCR builds can result - # in a race condition...i.e. the copy to DockerHub may fail if GHCR image has been updated - # at the moment when the copy occurs. - ########################################################################################## - docker-copy_to_dockerhub: - # Only run if this is NOT a PR - if: ${{ github.event_name != 'pull_request' }} - runs-on: ubuntu-latest - needs: - - docker-build_manifest - - steps: - # 'regctl' is used to more easily copy the image to DockerHub and obtain the digest from DockerHub - # See https://github.com/regclient/regclient/blob/main/docs/regctl.md - - name: Install regctl for Docker registry tools - run: | - export REGCTL_VERSION=v0.9.2 - mkdir -p bin - curl -sSLo bin/regctl https://github.com/regclient/regclient/releases/download/${REGCTL_VERSION}/regctl-linux-amd64 - chmod a+x bin/regctl - echo "$(pwd)/bin" >> $GITHUB_PATH - - # This recreates Docker tags for DockerHub - - name: Add Docker metadata for image - id: meta_dockerhub - uses: docker/metadata-action@v6 - with: - images: ${{ env.IMAGE_NAME }} - tags: ${{ env.IMAGE_TAGS }} - flavor: ${{ env.TAGS_FLAVOR }} - - # Login to source registry first, as this is where we are copying *from* - - name: Login to ${{ env.DOCKER_BUILD_REGISTRY }} - uses: docker/login-action@v4 - with: - registry: ${{ env.DOCKER_BUILD_REGISTRY }} - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - # Login to DockerHub, since this is where we are copying *to* - - name: Login to DockerHub - uses: docker/login-action@v4 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_ACCESS_TOKEN }} - - # Copy the image from source to DockerHub - - name: Copy image from ${{ env.DOCKER_BUILD_REGISTRY }} to docker.io - run: | - regctl image copy ${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta_dockerhub.outputs.version }} docker.io/${{ env.IMAGE_NAME }}:${{ steps.meta_dockerhub.outputs.version }} - - #-------------------------------------------------------------------- - # Finally, check whether demo.dspace.org or sandbox.dspace.org need - # to be redeployed based on these new DockerHub images. - #-------------------------------------------------------------------- - # If this build is for the branch that Sandbox uses and passed in a REDEPLOY_SANDBOX_URL secret, - # Then redeploy https://sandbox.dspace.org - - name: Redeploy sandbox.dspace.org (based on main branch) - if: | - env.REDEPLOY_SANDBOX_URL != '' && - github.ref_name == env.DEPLOY_SANDBOX_BRANCH - run: | - curl -X POST $REDEPLOY_SANDBOX_URL - # If this build is for the branch that Demo uses and passed in a REDEPLOY_DEMO_URL secret, - # Then redeploy https://demo.dspace.org - - name: Redeploy demo.dspace.org (based on maintenance branch) - if: | - env.REDEPLOY_DEMO_URL != '' && - github.ref_name == env.DEPLOY_DEMO_BRANCH - run: | - curl -X POST $REDEPLOY_DEMO_URL + cache-to: type=gha,scope=${{ inputs.build_id }},mode=max diff --git a/.github/workflows/tag-release.yml b/.github/workflows/tag-release.yml new file mode 100644 index 000000000000..6b2bf2eeedc8 --- /dev/null +++ b/.github/workflows/tag-release.yml @@ -0,0 +1,46 @@ +name: Release + +on: + push: + tags: + - '**' + workflow_dispatch: + + +env: + IMAGE_BASE_NAME: dataquest/dspace + +jobs: + retag-BE-image: + runs-on: ubuntu-latest + steps: + - name: Login to DockerHub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_ACCESS_TOKEN }} + + - name: retag image + run: | + docker pull ${{ env.IMAGE_BASE_NAME }}:${{ github.sha }} + docker tag ${{ env.IMAGE_BASE_NAME }}:${{ github.sha }} ${{ env.IMAGE_BASE_NAME }}:${{ github.ref_name }} + + docker pull ${{ env.IMAGE_BASE_NAME }}-cli:${{ github.sha }} + docker tag ${{ env.IMAGE_BASE_NAME }}-cli:${{ github.sha }} ${{ env.IMAGE_BASE_NAME }}-cli:${{ github.ref_name }} + + docker pull ${{ env.IMAGE_BASE_NAME }}-solr:${{ github.sha }} + docker tag ${{ env.IMAGE_BASE_NAME }}-solr:${{ github.sha }} ${{ env.IMAGE_BASE_NAME }}-solr:${{ github.ref_name }} + + docker pull ${{ env.IMAGE_BASE_NAME }}-postgres-pgcrypto:${{ github.sha }} + docker tag ${{ env.IMAGE_BASE_NAME }}-postgres-pgcrypto:${{ github.sha }} ${{ env.IMAGE_BASE_NAME }}-postgres-pgcrypto:${{ github.ref_name }} + + docker pull ${{ env.IMAGE_BASE_NAME }}-dependencies:${{ github.sha }} + docker tag ${{ env.IMAGE_BASE_NAME }}-dependencies:${{ github.sha }} ${{ env.IMAGE_BASE_NAME }}-dependencies:${{ github.ref_name }} + + - name: push image + run: | + docker push ${{ env.IMAGE_BASE_NAME }}:${{ github.ref_name }} + docker push ${{ env.IMAGE_BASE_NAME }}-cli:${{ github.ref_name }} + docker push ${{ env.IMAGE_BASE_NAME }}-solr:${{ github.ref_name }} + docker push ${{ env.IMAGE_BASE_NAME }}-postgres-pgcrypto:${{ github.ref_name }} + docker push ${{ env.IMAGE_BASE_NAME }}-dependencies:${{ github.ref_name }} diff --git a/Dockerfile b/Dockerfile index 668aa8fa591a..15ace9def419 100644 --- a/Dockerfile +++ b/Dockerfile @@ -67,5 +67,12 @@ RUN apt-get update \ EXPOSE 8080 8000 # Give java extra memory (2GB) ENV JAVA_OPTS=-Xmx2000m + +# We create a 'dspace' user to run DSpace instead of running as root. An explicit UID is required +# because Kubernetes deployment accepts only numeric user IDs when specifying the container user. +RUN useradd -u 1100 -m -s /bin/bash dspace \ + && chown -Rv dspace: /dspace +USER dspace + # On startup, run DSpace Runnable JAR (uses the "dspace.dir" setting defined in "dspace__P__dir" env variable) ENTRYPOINT ["java", "-jar", "webapps/server-boot.jar"] diff --git a/dspace/src/main/docker-compose/db.entities.yml b/dspace/src/main/docker-compose/db.entities.yml index 10ff4246b8c9..1e1aa7ebd73e 100644 --- a/dspace/src/main/docker-compose/db.entities.yml +++ b/dspace/src/main/docker-compose/db.entities.yml @@ -8,10 +8,10 @@ services: dspacedb: - image: "${DOCKER_REGISTRY:-docker.io}/${DOCKER_OWNER:-dspace}/dspace-postgres-loadsql:${DSPACE_VER:-dspace-9_x}" + image: dataquest/dspace-postgres-pgcrypto:${DSPACE_VER:-dspace-9_x}-loadsql environment: # This SQL is available from https://github.com/DSpace-Labs/AIP-Files/releases/tag/demo-entities-data - - LOADSQL=${LOADSQL:-https://github.com/DSpace-Labs/AIP-Files/releases/download/demo-entities-data/dspace7-entities-data.sql} + - LOADSQL=https://github.com/DSpace-Labs/AIP-Files/releases/download/demo-entities-data/dspace7-entities-data.sql dspace: ### OVERRIDE default 'entrypoint' in 'docker-compose.yml #### # Ensure that the database is ready BEFORE starting tomcat @@ -24,4 +24,4 @@ services: - | while (! /dev/null 2>&1; do sleep 1; done; /dspace/bin/dspace database migrate ignored - java -jar /dspace/webapps/server-boot.jar + java -jar /dspace/webapps/server-boot.jar --dspace.dir=/dspace diff --git a/dspace/src/main/docker-compose/db.restore.yml b/dspace/src/main/docker-compose/db.restore.yml index 4fb4f025efd4..4d8e1aa5f674 100644 --- a/dspace/src/main/docker-compose/db.restore.yml +++ b/dspace/src/main/docker-compose/db.restore.yml @@ -12,10 +12,10 @@ # This can be used to restore a "dspacedb" container from a pg_dump, or during upgrade to a new version of PostgreSQL. services: dspacedb: - image: "${DOCKER_REGISTRY:-docker.io}/${DOCKER_OWNER:-dspace}/dspace-postgres-loadsql:${DSPACE_VER:-dspace-9_x}" + image: dataquest/dspace-postgres-pgcrypto:${DSPACE_VER:-dspace-9_x}-loadsql environment: # Location where the dump SQL file will be available on the running container - - LOCALSQL=${LOCALSQL:-/tmp/pgdump.sql} + - LOCALSQL=/tmp/pgdump.sql volumes: # Volume which shares a local SQL file at "./pgdump.sql" to the running container # IF YOUR LOCAL FILE HAS A DIFFERENT NAME (or is in a different location), then change the "./pgdump.sql" diff --git a/dspace/src/main/docker/dspace-postgres-loadsql/Dockerfile b/dspace/src/main/docker/dspace-postgres-loadsql/Dockerfile deleted file mode 100644 index 1641970ccf3b..000000000000 --- a/dspace/src/main/docker/dspace-postgres-loadsql/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -# -# The contents of this file are subject to the license and copyright -# detailed in the LICENSE and NOTICE files at the root of the source -# tree and available online at -# -# http://www.dspace.org/license/ -# - -# Example build: -# docker build --build-arg POSTGRES_VERSION=17 --build-arg POSTGRES_PASSWORD=mypass ./dspace/src/main/docker/dspace-postgres-loadsql/ -# This will be published as dspace/dspace-postgres-loadsql:$DSPACE_VERSION - -ARG POSTGRES_VERSION=15 -ARG POSTGRES_DB=dspace -ARG POSTGRES_USER=dspace -ARG POSTGRES_PASSWORD=dspace - -FROM docker.io/postgres:${POSTGRES_VERSION} - -ENV POSTGRES_DB=${POSTGRES_DB} -ENV POSTGRES_USER=${POSTGRES_USER} -ENV POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - -# Install curl and ca-certificates which are necessary to load SQL file -RUN apt-get update \ - && apt-get install -y --no-install-recommends curl ca-certificates \ - && apt-get purge -y --auto-remove \ - && rm -rf /var/lib/apt/lists/* - -# Load a SQL dump. Set LOADSQL to a URL for the sql dump file. -COPY loadsql.sh /docker-entrypoint-initdb.d/ diff --git a/dspace/src/main/docker/dspace-postgres-pgcrypto-curl/Dockerfile b/dspace/src/main/docker/dspace-postgres-pgcrypto-curl/Dockerfile new file mode 100644 index 000000000000..f314fdb79e80 --- /dev/null +++ b/dspace/src/main/docker/dspace-postgres-pgcrypto-curl/Dockerfile @@ -0,0 +1,26 @@ +# +# The contents of this file are subject to the license and copyright +# detailed in the LICENSE and NOTICE files at the root of the source +# tree and available online at +# +# http://www.dspace.org/license/ +# + +# To build for example use: +# docker build --build-arg POSTGRES_VERSION=13 --build-arg POSTGRES_PASSWORD=mypass ./dspace/src/main/docker/dspace-postgres-pgcrypto-curl/ +# This will be published as dspace/dspace-postgres-pgcrypto:$DSPACE_VERSION-loadsql + +ARG POSTGRES_VERSION=15 +ARG POSTGRES_PASSWORD=dspace + +FROM docker.io/postgres:${POSTGRES_VERSION} + +ENV POSTGRES_DB=dspace +ENV POSTGRES_USER=dspace +ENV POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + +# Install curl which is necessary to load SQL file +RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/* + +# Load a SQL dump. Set LOADSQL to a URL for the sql dump file. +COPY install-pgcrypto.sh /docker-entrypoint-initdb.d/ diff --git a/dspace/src/main/docker/dspace-postgres-loadsql/loadsql.sh b/dspace/src/main/docker/dspace-postgres-pgcrypto-curl/install-pgcrypto.sh similarity index 58% rename from dspace/src/main/docker/dspace-postgres-loadsql/loadsql.sh rename to dspace/src/main/docker/dspace-postgres-pgcrypto-curl/install-pgcrypto.sh index 1639e6f8094d..d8e0382010df 100644 --- a/dspace/src/main/docker/dspace-postgres-loadsql/loadsql.sh +++ b/dspace/src/main/docker/dspace-postgres-pgcrypto-curl/install-pgcrypto.sh @@ -34,3 +34,16 @@ then touch $CHECKFILE fi + +# Then, setup pgcrypto on this database +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL + -- Create a new schema in this database named "extensions" (or whatever you want to name it) + CREATE SCHEMA IF NOT EXISTS extensions; + -- Enable this extension in this new schema + CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA extensions; + -- Update your database's "search_path" to also search the new "extensions" schema. + -- You are just appending it on the end of the existing comma-separated list. + ALTER DATABASE dspace SET search_path TO "\$user",public,extensions; + -- Grant rights to call functions in the extensions schema to your dspace user + GRANT USAGE ON SCHEMA extensions TO $POSTGRES_USER; +EOSQL diff --git a/dspace/src/main/docker/dspace-postgres-pgcrypto/Dockerfile b/dspace/src/main/docker/dspace-postgres-pgcrypto/Dockerfile new file mode 100644 index 000000000000..27935ac01d0d --- /dev/null +++ b/dspace/src/main/docker/dspace-postgres-pgcrypto/Dockerfile @@ -0,0 +1,25 @@ +# +# The contents of this file are subject to the license and copyright +# detailed in the LICENSE and NOTICE files at the root of the source +# tree and available online at +# +# http://www.dspace.org/license/ +# + +# To build for example use: +# docker build --build-arg POSTGRES_VERSION=13 --build-arg POSTGRES_PASSWORD=mypass ./dspace/src/main/docker/dspace-postgres-pgcrypto/ +# This will be published as dspace/dspace-postgres-pgcrypto:$DSPACE_VERSION + +ARG POSTGRES_VERSION=15 +ARG POSTGRES_PASSWORD=dspace + +FROM docker.io/postgres:${POSTGRES_VERSION} + +ENV POSTGRES_DB=dspace +ENV POSTGRES_USER=dspace +ENV POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + +# Create the log directory with the correct permissions +RUN mkdir -p /var/log/postgresql && chown postgres:postgres /var/log/postgresql + +COPY install-pgcrypto.sh /docker-entrypoint-initdb.d/ diff --git a/dspace/src/main/docker/dspace-postgres-pgcrypto/install-pgcrypto.sh b/dspace/src/main/docker/dspace-postgres-pgcrypto/install-pgcrypto.sh new file mode 100644 index 000000000000..67c4539b5a46 --- /dev/null +++ b/dspace/src/main/docker/dspace-postgres-pgcrypto/install-pgcrypto.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# +# The contents of this file are subject to the license and copyright +# detailed in the LICENSE and NOTICE files at the root of the source +# tree and available online at +# +# http://www.dspace.org/license/ +# + +set -e + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL + -- Create a new schema in this database named "extensions" (or whatever you want to name it) + CREATE SCHEMA IF NOT EXISTS extensions; + -- Enable this extension in this new schema + CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA extensions; + -- Update your database's "search_path" to also search the new "extensions" schema. + -- You are just appending it on the end of the existing comma-separated list. + ALTER DATABASE dspace SET search_path TO "\$user",public,extensions; + -- Grant rights to call functions in the extensions schema to your dspace user + GRANT USAGE ON SCHEMA extensions TO $POSTGRES_USER; +EOSQL diff --git a/scripts/sourceversion.py b/scripts/sourceversion.py new file mode 100644 index 000000000000..b838d70708c8 --- /dev/null +++ b/scripts/sourceversion.py @@ -0,0 +1,22 @@ +import subprocess +import sys +from datetime import datetime, timezone + +def get_time_in_timezone(zone: str = "Europe/Bratislava"): + try: + from zoneinfo import ZoneInfo + my_tz = ZoneInfo(zone) + except Exception as e: + my_tz = timezone.utc + return datetime.now(my_tz) + + +if __name__ == '__main__': + ts = get_time_in_timezone() + print(f"This info was generated on: {ts.strftime('%Y-%m-%d %H:%M:%S %Z%z')}") + + cmd = 'git log -1 --pretty=format:"Git hash: %H Date of commit: %ai"' + subprocess.check_call(cmd, shell=True) + + link = sys.argv[1] + sys.argv[2] + print(' Build run: ' + link + ' ')