Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .dockerignore
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
*/target/
dspace/modules/*/target/
Dockerfile.*
dspace/src/main/docker/dspace-postgres-loadsql
dspace/src/main/docker/dspace-postgres-pgcrypto
dspace/src/main/docker/dspace-postgres-pgcrypto-curl
dspace/src/main/docker/README.md
dspace/src/main/docker-compose/
13 changes: 8 additions & 5 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,14 @@
# https://docs.github.com/en/free-pro-team@latest/actions/guides/building-and-testing-java-with-maven
name: Build

# Run this Build for all pushes / PRs to current branch
on: [push, pull_request]
# Run this Build for pushes to our main and all PRs
on:
push:
branches:
- dtq-dev
- customer/*
pull_request:
Comment thread
Kasinhou marked this conversation as resolved.
workflow_dispatch:

permissions:
contents: read # to fetch code (actions/checkout)
Expand Down Expand Up @@ -81,9 +87,6 @@ jobs:
# Codecov upload is a separate job in order to allow us to restart this separate from the entire build/test
# job above. This is necessary because Codecov uploads seem to randomly fail at times.
# See https://community.codecov.com/t/upload-issues-unable-to-locate-build-via-github-actions-api/3954
# Codecov upload disabled for the JCU fork: the repository is not configured with a
# CODECOV_TOKEN and the project-level coverage check fails on customization commits.
# Coverage is still produced and uploaded as a build artifact by the 'tests' job above.
# codecov:
# # Must run after 'tests' job above
# needs: tests
Expand Down
224 changes: 74 additions & 150 deletions .github/workflows/docker.yml

Large diffs are not rendered by default.

233 changes: 30 additions & 203 deletions .github/workflows/reusable-docker-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,14 @@ on:
workflow_call:
# Possible Inputs to this reusable job
inputs:
python_version_script_dest:
required: false
default: version.txt
type: string
run_python_version_script:
required: false
default: false
type: boolean
# Build name/id for this Docker build. Used for digest storage to avoid digest overlap between builds.
build_id:
required: true
Expand Down Expand Up @@ -58,9 +66,9 @@ env:
# e.g. PR against 'main' will use "latest". a PR against 'dspace-7_x' will use 'dspace-7_x'.
IMAGE_TAGS: |
type=raw,value=latest,enable=${{ github.ref_name == github.event.repository.default_branch }}
type=raw,value=${{ github.sha }}
type=ref,event=branch,enable=${{ github.ref_name != github.event.repository.default_branch }}
type=ref,event=tag
type=raw,value=${{ (github.event.pull_request.base.ref == github.event.repository.default_branch && 'latest') || github.event.pull_request.base.ref }},enable=${{ github.event_name == 'pull_request' }}
# Define default tag "flavor" for docker/metadata-action per
# https://github.com/docker/metadata-action#flavor-input
# We manage the 'latest' tag ourselves to the 'main' branch (see settings above)
Expand All @@ -77,25 +85,27 @@ env:
DEPLOY_ARCH: 'linux/amd64'
# Registry used during building of Docker images. (All images are later copied to docker.io registry)
# We use GitHub's Container Registry to avoid aggressive rate limits at DockerHub.
DOCKER_BUILD_REGISTRY: ghcr.io
DOCKER_BUILD_REGISTRY: docker.io

jobs:
docker-build:

strategy:
matrix:
# Architectures / Platforms for which we will build Docker images
arch: [ 'linux/amd64', 'linux/arm64' ]
# arch: [ 'linux/amd64', 'linux/arm64' ]
arch: [ 'linux/amd64' ]
os: [ ubuntu-latest ]
isPr:
- ${{ github.event_name == 'pull_request' }}
# If this is a PR, we ONLY build for AMD64. For PRs we only do a sanity check test to ensure Docker builds work.
# The below exclude therefore ensures we do NOT build ARM64 for PRs.
exclude:
- isPr: true
arch: linux/arm64
# If this is a PR, we ONLY build for AMD64. For PRs we only do a sanity check test to ensure Docker builds work.
# The below exclude therefore ensures we do NOT build ARM64 for PRs.
# exclude:
# - isPr: true
# os: ubuntu-latest
# arch: linux/arm64

# If ARM64, then use the Ubuntu ARM64 runner. Otherwise, use the Ubuntu AMD64 runner
runs-on: ${{ matrix.arch == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
runs-on: ${{ matrix.os }}

steps:
# This step converts the slashes in the "arch" matrix values above into dashes & saves to env.ARCH_NAME
Expand All @@ -111,15 +121,21 @@ jobs:
- name: Checkout codebase
uses: actions/checkout@v6

- name: Add version
if: ${{ inputs.run_python_version_script }}
run: python scripts/sourceversion.py ${{ github.server_url }}/${{ github.repository }}/actions/runs/ ${{ github.run_id }} > ${{ inputs.python_version_script_dest }}


# https://github.com/docker/login-action
# NOTE: This login occurs for BOTH non-PRs or PRs. PRs *must* also login to access private images from GHCR
# during the build process
- name: Login to ${{ env.DOCKER_BUILD_REGISTRY }}
# Only login if not a PR, as PRs only trigger a Docker build and not a push
if: ${{ ! matrix.isPr }}
uses: docker/login-action@v4
with:
registry: ${{ env.DOCKER_BUILD_REGISTRY }}
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_ACCESS_TOKEN }}

# https://github.com/docker/setup-buildx-action
- name: Setup Docker Buildx
Expand Down Expand Up @@ -164,193 +180,4 @@ jobs:
# Use GitHub cache to load cached Docker images and cache the results of this build
# This decreases the number of images we need to fetch from DockerHub
cache-from: type=gha,scope=${{ inputs.build_id }}
cache-to: type=gha,scope=${{ inputs.build_id }},mode=min

# Export the digest of Docker build locally
- name: Export Docker build digest
if: ${{ ! matrix.isPr }}
run: |
mkdir -p /tmp/digests
digest="${{ steps.docker_build.outputs.digest }}"
touch "/tmp/digests/${digest#sha256:}"

# Upload digest to an artifact, so that it can be used in combined manifest below
# (The purpose of the combined manifest is to list both amd64 and arm64 builds under same tag)
- name: Upload Docker build digest to artifact
if: ${{ ! matrix.isPr }}
uses: actions/upload-artifact@v7
with:
name: digests-${{ inputs.build_id }}-${{ env.ARCH_NAME }}
path: /tmp/digests/*
if-no-files-found: error
retention-days: 1

#------------------------------------------------------------------------------
# Second, we build the image again in order to store it in a local TAR file.
# This TAR of the image is cached/saved as an artifact, so that it can be used
# by later jobs to install the brand-new images for automated testing.
# This TAR build is performed BOTH for PRs and for branch commits (non-PRs).
#
# (This approach has the advantage of avoiding having to download the newly built
# image from DockerHub or GHCR during automated testing.)
#
# See the 'docker-deploy' job in docker.yml as an example of where this TAR is used.
#-------------------------------------------------------------------------------
# Build local image (again) and store in a TAR file in /tmp directory
# This step is only done for AMD64, as that's the only image we use in our automated testing (at this time).
# NOTE: This step cannot be combined with the build above as it's a different type of output.
- name: Build and push image to local TAR file
if: ${{ matrix.arch == 'linux/amd64'}}
uses: docker/build-push-action@v7
with:
build-contexts: |
${{ inputs.dockerfile_additional_contexts }}
context: ${{ inputs.dockerfile_context }}
file: ${{ inputs.dockerfile_path }}
# Tell DSpace's Docker files to use the build registry instead of DockerHub
build-args:
DOCKER_REGISTRY=${{ env.DOCKER_BUILD_REGISTRY }}
platforms: ${{ matrix.arch }}
tags: ${{ steps.meta_build.outputs.tags }}
labels: ${{ steps.meta_build.outputs.labels }}
# Use GitHub cache to load cached Docker images and cache the results of this build
# This decreases the number of images we need to fetch from DockerHub
cache-from: type=gha,scope=${{ inputs.build_id }}
cache-to: type=gha,scope=${{ inputs.build_id }},mode=min
# Export image to a local TAR file
outputs: type=docker,dest=/tmp/${{ inputs.build_id }}.tar

# Upload the local docker image (in TAR file) to a build Artifact
# This step is only done for AMD64, as that's the only image we use in our automated testing (at this time).
- name: Upload local image TAR to artifact
if: ${{ matrix.arch == 'linux/amd64'}}
uses: actions/upload-artifact@v7
with:
name: docker-image-${{ inputs.build_id }}-${{ env.ARCH_NAME }}
path: /tmp/${{ inputs.build_id }}.tar
if-no-files-found: error
retention-days: 1

##########################################################################################
# Merge Docker digests (from various architectures) into a single manifest.
# This runs after all Docker builds complete above. The purpose is to include all builds
# under a single manifest for this tag.
# (e.g. both linux/amd64 and linux/arm64 should be listed under the same tagged Docker image)
##########################################################################################
docker-build_manifest:
# Only run if this is NOT a PR
if: ${{ github.event_name != 'pull_request' }}
runs-on: ubuntu-latest
needs:
- docker-build
steps:
- name: Download Docker build digests
uses: actions/download-artifact@v8
with:
path: /tmp/digests
# Download digests for both AMD64 and ARM64 into same directory
pattern: digests-${{ inputs.build_id }}-*
merge-multiple: true

- name: Login to ${{ env.DOCKER_BUILD_REGISTRY }}
uses: docker/login-action@v4
with:
registry: ${{ env.DOCKER_BUILD_REGISTRY }}
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4

- name: Add Docker metadata for image
id: meta
uses: docker/metadata-action@v6
with:
images: ${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }}
tags: ${{ env.IMAGE_TAGS }}
flavor: ${{ env.TAGS_FLAVOR }}

- name: Create manifest list from digests and push to ${{ env.DOCKER_BUILD_REGISTRY }}
working-directory: /tmp/digests
run: |
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
$(printf '${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)

- name: Inspect manifest in ${{ env.DOCKER_BUILD_REGISTRY }}
run: |
docker buildx imagetools inspect ${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}

##########################################################################################
# Copy images / manifest to DockerHub.
# This MUST run after *both* images (AMD64 and ARM64) are built and uploaded to GitHub
# Container Registry (GHCR). Attempting to run this in parallel to GHCR builds can result
# in a race condition...i.e. the copy to DockerHub may fail if GHCR image has been updated
# at the moment when the copy occurs.
##########################################################################################
docker-copy_to_dockerhub:
# Only run if this is NOT a PR
if: ${{ github.event_name != 'pull_request' }}
runs-on: ubuntu-latest
needs:
- docker-build_manifest

steps:
# 'regctl' is used to more easily copy the image to DockerHub and obtain the digest from DockerHub
# See https://github.com/regclient/regclient/blob/main/docs/regctl.md
- name: Install regctl for Docker registry tools
run: |
export REGCTL_VERSION=v0.9.2
mkdir -p bin
curl -sSLo bin/regctl https://github.com/regclient/regclient/releases/download/${REGCTL_VERSION}/regctl-linux-amd64
chmod a+x bin/regctl
echo "$(pwd)/bin" >> $GITHUB_PATH

# This recreates Docker tags for DockerHub
- name: Add Docker metadata for image
id: meta_dockerhub
uses: docker/metadata-action@v6
with:
images: ${{ env.IMAGE_NAME }}
tags: ${{ env.IMAGE_TAGS }}
flavor: ${{ env.TAGS_FLAVOR }}

# Login to source registry first, as this is where we are copying *from*
- name: Login to ${{ env.DOCKER_BUILD_REGISTRY }}
uses: docker/login-action@v4
with:
registry: ${{ env.DOCKER_BUILD_REGISTRY }}
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}

# Login to DockerHub, since this is where we are copying *to*
- name: Login to DockerHub
uses: docker/login-action@v4
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_ACCESS_TOKEN }}

# Copy the image from source to DockerHub
- name: Copy image from ${{ env.DOCKER_BUILD_REGISTRY }} to docker.io
run: |
regctl image copy ${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta_dockerhub.outputs.version }} docker.io/${{ env.IMAGE_NAME }}:${{ steps.meta_dockerhub.outputs.version }}

#--------------------------------------------------------------------
# Finally, check whether demo.dspace.org or sandbox.dspace.org need
# to be redeployed based on these new DockerHub images.
#--------------------------------------------------------------------
# If this build is for the branch that Sandbox uses and passed in a REDEPLOY_SANDBOX_URL secret,
# Then redeploy https://sandbox.dspace.org
- name: Redeploy sandbox.dspace.org (based on main branch)
if: |
env.REDEPLOY_SANDBOX_URL != '' &&
github.ref_name == env.DEPLOY_SANDBOX_BRANCH
run: |
curl -X POST $REDEPLOY_SANDBOX_URL
# If this build is for the branch that Demo uses and passed in a REDEPLOY_DEMO_URL secret,
# Then redeploy https://demo.dspace.org
- name: Redeploy demo.dspace.org (based on maintenance branch)
if: |
env.REDEPLOY_DEMO_URL != '' &&
github.ref_name == env.DEPLOY_DEMO_BRANCH
run: |
curl -X POST $REDEPLOY_DEMO_URL
cache-to: type=gha,scope=${{ inputs.build_id }},mode=max
46 changes: 46 additions & 0 deletions .github/workflows/tag-release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
name: Release

on:
push:
tags:
- '**'
workflow_dispatch:


env:
IMAGE_BASE_NAME: dataquest/dspace

jobs:
retag-BE-image:
runs-on: ubuntu-latest
steps:
- name: Login to DockerHub
uses: docker/login-action@v4
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_ACCESS_TOKEN }}

- name: retag image
run: |
docker pull ${{ env.IMAGE_BASE_NAME }}:${{ github.sha }}
docker tag ${{ env.IMAGE_BASE_NAME }}:${{ github.sha }} ${{ env.IMAGE_BASE_NAME }}:${{ github.ref_name }}

docker pull ${{ env.IMAGE_BASE_NAME }}-cli:${{ github.sha }}
docker tag ${{ env.IMAGE_BASE_NAME }}-cli:${{ github.sha }} ${{ env.IMAGE_BASE_NAME }}-cli:${{ github.ref_name }}

docker pull ${{ env.IMAGE_BASE_NAME }}-solr:${{ github.sha }}
docker tag ${{ env.IMAGE_BASE_NAME }}-solr:${{ github.sha }} ${{ env.IMAGE_BASE_NAME }}-solr:${{ github.ref_name }}

docker pull ${{ env.IMAGE_BASE_NAME }}-postgres-pgcrypto:${{ github.sha }}
docker tag ${{ env.IMAGE_BASE_NAME }}-postgres-pgcrypto:${{ github.sha }} ${{ env.IMAGE_BASE_NAME }}-postgres-pgcrypto:${{ github.ref_name }}

docker pull ${{ env.IMAGE_BASE_NAME }}-dependencies:${{ github.sha }}
docker tag ${{ env.IMAGE_BASE_NAME }}-dependencies:${{ github.sha }} ${{ env.IMAGE_BASE_NAME }}-dependencies:${{ github.ref_name }}

- name: push image
run: |
docker push ${{ env.IMAGE_BASE_NAME }}:${{ github.ref_name }}
docker push ${{ env.IMAGE_BASE_NAME }}-cli:${{ github.ref_name }}
docker push ${{ env.IMAGE_BASE_NAME }}-solr:${{ github.ref_name }}
docker push ${{ env.IMAGE_BASE_NAME }}-postgres-pgcrypto:${{ github.ref_name }}
docker push ${{ env.IMAGE_BASE_NAME }}-dependencies:${{ github.ref_name }}
Comment thread
Kasinhou marked this conversation as resolved.
7 changes: 7 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -67,5 +67,12 @@ RUN apt-get update \
EXPOSE 8080 8000
# Give java extra memory (2GB)
ENV JAVA_OPTS=-Xmx2000m

# We create a 'dspace' user to run DSpace instead of running as root. An explicit UID is required
# because Kubernetes deployment accepts only numeric user IDs when specifying the container user.
RUN useradd -u 1100 -m -s /bin/bash dspace \
&& chown -Rv dspace: /dspace
Comment thread
Kasinhou marked this conversation as resolved.
USER dspace

# On startup, run DSpace Runnable JAR (uses the "dspace.dir" setting defined in "dspace__P__dir" env variable)
ENTRYPOINT ["java", "-jar", "webapps/server-boot.jar"]
6 changes: 3 additions & 3 deletions dspace/src/main/docker-compose/db.entities.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@

services:
dspacedb:
image: "${DOCKER_REGISTRY:-docker.io}/${DOCKER_OWNER:-dspace}/dspace-postgres-loadsql:${DSPACE_VER:-dspace-9_x}"
image: dataquest/dspace-postgres-pgcrypto:${DSPACE_VER:-dspace-9_x}-loadsql
environment:
# This SQL is available from https://github.com/DSpace-Labs/AIP-Files/releases/tag/demo-entities-data
- LOADSQL=${LOADSQL:-https://github.com/DSpace-Labs/AIP-Files/releases/download/demo-entities-data/dspace7-entities-data.sql}
- LOADSQL=https://github.com/DSpace-Labs/AIP-Files/releases/download/demo-entities-data/dspace7-entities-data.sql
dspace:
### OVERRIDE default 'entrypoint' in 'docker-compose.yml ####
# Ensure that the database is ready BEFORE starting tomcat
Expand All @@ -24,4 +24,4 @@ services:
- |
while (!</dev/tcp/dspacedb/5432) > /dev/null 2>&1; do sleep 1; done;
/dspace/bin/dspace database migrate ignored
java -jar /dspace/webapps/server-boot.jar
java -jar /dspace/webapps/server-boot.jar --dspace.dir=/dspace
Loading