diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md deleted file mode 100644 index 7b5dc0b3487..00000000000 --- a/.github/CONTRIBUTING.md +++ /dev/null @@ -1 +0,0 @@ -Make sure that you generate site HTML with `jekyll build`, and include the changes to the HTML in your pull request also. See README.md for more information. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index 9e1ffb348a5..00000000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1 +0,0 @@ - diff --git a/.github/actions/build-html/action.yml b/.github/actions/build-html/action.yml new file mode 100644 index 00000000000..cc8e02d5fa9 --- /dev/null +++ b/.github/actions/build-html/action.yml @@ -0,0 +1,22 @@ +name: Build HTML + +description: Set up Ruby and run the Jekyll build. + +inputs: + ruby-version: + description: Ruby version to use. + required: false + default: "3.4" + +runs: + using: composite + steps: + - name: Set up Ruby and Bundler + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ inputs.ruby-version }} + # This will use the version of Bundler specified in `Gemfile.lock`. + bundler-cache: true + - name: Run documentation build + shell: bash + run: bundle exec jekyll build diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000000..00865c5c6c7 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,5 @@ + diff --git a/.github/workflows/doc_gen.yml b/.github/workflows/doc_gen.yml deleted file mode 100644 index 01478f92735..00000000000 --- a/.github/workflows/doc_gen.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: Check document generation - -on: - push: - branches: - - asf-site - pull_request: - branches: - - asf-site - -jobs: - lint: - name: check whether all documentation was generated with the right Jekyll version - runs-on: ubuntu-24.04 - steps: - - name: Free up disk space - shell: 'script -q -e -c "bash {0}"' - run: | - echo "==================================" - echo "Free up disk space on CI system" - echo "==================================" - - echo "Listing top 100 largest packages (from large to small)" - printf "Installed-Size\tPackage\n" - dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n -r | head -n 100 - df -h - - echo "Removing large packages" - rm -rf /__t/CodeQL - rm -rf /__t/go - rm -rf /__t/node - - apt-get remove --purge -y '^aspnet.*' || true - apt-get remove --purge -y '^dotnet-.*' || true - apt-get remove --purge -y '^llvm-.*' || true - apt-get remove --purge -y 'php.*' || true - apt-get remove --purge -y '^mongodb-.*' || true - apt-get remove --purge -y 'gfortran-11' || true - apt-get remove --purge -y 'humanity-icon-theme' || true - apt-get remove --purge -y 'nodejs-doc' || true - apt-get remove --purge -y snapd google-chrome-stable microsoft-edge-stable firefox || true - apt-get remove --purge -y azure-cli google-cloud-sdk mono-devel powershell libgl1-mesa-dri || true - apt-get autoremove --purge -y - apt-get clean - - df -h - - name: Checkout Spark Website repository - uses: actions/checkout@v2 - - name: Install dependencies for documentation generation - run: | - sudo apt-get update -y - sudo apt-get install -y ruby ruby-dev - sudo gem install bundler --version 2.4.19 - bundle install - - name: Run documentation build - run: | - export LC_ALL=C.UTF-8 - export LANG=C.UTF-8 - OLD_IFS=$IFS - IFS= - GEN_ERRORS=$(bundle exec jekyll build 3>&2 2>&1 1>&3) - if [ $(echo $GEN_ERRORS| grep -v -e '^$'| grep -c -v "rubygems_integration") -ne 0 ]; then - echo "Error during document generation:" - echo $GEN_ERRORS - exit 1 - fi - IFS=$OLD_IFS - CHANGED_FILE=( $(git ls-files --modified --other --exclude-standard --directory | grep -v sitemap.xml | grep -v llms.txt) ) - if [ ${#CHANGED_FILE[@]} -ne 0 ]; then - echo "Not all documentation was generated and/or not the right Jekyll version was used! Modified / untracked files (excluding sitemap.xml):" - echo ${CHANGED_FILE[*]} - echo "Git diff (excluding sitemap.xml):" - git diff -- . ':(exclude)site/sitemap.xml' - exit 1 - fi - shell: /bin/bash {0} diff --git a/.github/workflows/html-build.yml b/.github/workflows/html-build.yml new file mode 100644 index 00000000000..b94dfebe2b3 --- /dev/null +++ b/.github/workflows/html-build.yml @@ -0,0 +1,16 @@ +name: Build HTML + +on: + pull_request: + branches: + - asf-site + +jobs: + build: + name: Build HTML + runs-on: ubuntu-24.04 + steps: + - name: Checkout Spark Website repository + uses: actions/checkout@v7 + - name: Build HTML + uses: ./.github/actions/build-html diff --git a/.github/workflows/html-push.yml b/.github/workflows/html-push.yml new file mode 100644 index 00000000000..15160e7b4cd --- /dev/null +++ b/.github/workflows/html-push.yml @@ -0,0 +1,43 @@ +name: Build and Push HTML + +on: + push: + branches: + - asf-site + +jobs: + commit: + name: Build and commit HTML to `asf-site` + # This condition is important. We don't want to trigger this job if the last + # commit was created _by_ this job! + if: >- + !( + contains(github.event.head_commit.message, '[html]') && + github.event.head_commit.author.name == 'github-actions[bot]' + ) + # Not technically necessary, but helps avoid spurious failures if multiple + # commits are pushed in rapid succession. + concurrency: + group: html-push-${{ github.ref }} + cancel-in-progress: true + runs-on: ubuntu-24.04 + permissions: + contents: write + steps: + - name: Checkout Spark Website repository + uses: actions/checkout@v7 + - name: Build HTML + uses: ./.github/actions/build-html + - name: Commit and push generated HTML + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + # `-f` because we told git to otherwise ignore `site/` + git add -f site/ + if git diff --cached --quiet; then + echo "No changes to commit." + else + COMMIT_TITLE=$(git log -1 --pretty=%s) + git commit -m "[html] $COMMIT_TITLE" + git push + fi diff --git a/.gitignore b/.gitignore index d0d5f1c2673..90ef9e92051 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,4 @@ target/ .jekyll-cache/ .jekyll-metadata .local_ruby_bundle -site/python +site/ diff --git a/Gemfile.lock b/Gemfile.lock index 480f64554a0..de6896d5be8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -79,4 +79,4 @@ RUBY VERSION ruby 3.2.3p157 BUNDLED WITH - 2.4.19 + 2.4.22 diff --git a/README.md b/README.md index 4d4106f2c26..371cb7fdf0d 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,34 @@ -## Generating the website HTML +# Apache Spark Main Website -In this directory you will find text files formatted using Markdown, with an `.md` suffix. +This repository captures the main Apache Spark website located at https://spark.apache.org. The programming docs under https://spark.apache.org/docs/ are [in the main Spark repo][main], not here. They are built separately for each release of Spark and then copied to the website under the `docs/` directory. -Building the site requires [Ruby 3](https://www.ruby-lang.org), [Jekyll](http://jekyllrb.com/docs), and -[Rouge](https://github.com/rouge-ruby/rouge). The most reliable way to ensure a compatible environment -is to use the official Docker build image from the Apache Spark repository. +[main]: https://github.com/apache/spark/tree/master/docs#readme + +## Contributing + +To contribute changes, build and test the site locally, then submit a pull request with your changes. You only need to commit changes to the Markdown source. A [GitHub Actions workflow](.github/workflows/html-push.yml) will generate the corresponding HTML under `site/` and push it for you. + +The `site/` directory is tracked in git as a deployment artifact. To hide local changes to it from `git status` and `git diff`, run: + +```sh +git ls-files -z site/ | xargs -0 git update-index --skip-worktree +``` + +To undo this, run the same command with `--no-skip-worktree` instead. + +## Building the site locally + +Building the site requires [Ruby 3](https://www.ruby-lang.org), [Jekyll](http://jekyllrb.com/docs), and [Rouge](https://github.com/rouge-ruby/rouge). + +``` +gem install bundler -v 2.4.22 +bundle install +bundle exec jekyll serve +``` + +### Building the site with Docker + +The most reliable way to ensure a compatible environment is to use the official Docker build image from the Apache Spark repository. If you haven't already, clone the [Apache Spark](https://github.com/apache/spark) repository. Navigate to the Spark root directory and run the following command to create the builder image: @@ -21,25 +45,12 @@ the Markdown files in the Docker container. .dev/build-docs.sh ``` -## Docs sub-dir - -The docs are not generated as part of the website. They are built separately for each release -of Spark from the Spark source repository and then copied to the website under the docs -directory. See the instructions for building those in the readme in the Spark -project's `/docs` directory. - -## Rouge and Pygments - -We also use [Rouge](https://github.com/rouge-ruby/rouge) for syntax highlighting in documentation Markdown pages. -Its HTML output is compatible with CSS files designed for [Pygments](https://pygments.org/). +## Deploying to production -To mark a block of code in your Markdown to be syntax highlighted by `jekyll` during the -compile phase, use the following syntax: +The website is deployed automatically by [ASF Infra][infra]. The deployment configuration is tracked by [.asf.yaml](./.asf.yaml) and is [documented here][asf-docs]. - {% highlight scala %} - // Your Scala code goes here, you can replace Scala with many other - // supported languages too. - {% endhighlight %} +One deployment detail that appears to be critical is the presence of the [`content`](./content/) symlink to `site/`. Even though ASF Infra is [aware of Jekyll][jek], we perhaps do not have the exact setup required for them to automatically use our [Jekyll config](./_config.yml) to understand where the site content lives. Without the `content` symlink, the website will just show a plain directory listing of the files in this repo. -You probably don't need to install that unless you want to regenerate the Pygments CSS file. -It requires Python, and can be installed by running `sudo easy_install Pygments`. +[infra]: https://infra.apache.org +[asf-docs]: https://github.com/apache/infrastructure-asfyaml/tree/main#readme +[jek]: https://github.com/apache/infrastructure-asfyaml/tree/76d241ccef02e5397e10c173ebf04c07525311ea#jekyll_cms diff --git a/release-process.md b/release-process.md index 349edf6f3b5..edfb8057f41 100644 --- a/release-process.md +++ b/release-process.md @@ -365,8 +365,9 @@ Next, update the rest of the Spark website. See how the previous releases are do * add the new release to `site/static/versions.json` (attention to the order of releases) [for `spark version drop down` of the `PySpark` docs] * check `security.md` for anything to update -``` -$ git add 1.1.1 +```sh +# The `-f` is because we normally ignore changes under `site/`. +$ git add -f 1.1.1 $ git commit -m "Add docs for Spark 1.1.1" ``` @@ -377,9 +378,7 @@ pick the release version from the list, then click on "Release Notes". Copy this `spark-2.1.2`. Create a new release post under `releases/_posts` to include this short URL. The date of the post should be the date you create it. -Then run `bundle exec jekyll build` to update the `site` directory. - -Considering the Pull Request will be large, please separate the commits of code changes and generated `site` directory for an easier review. +You only need to commit and push changes to the website source. A GitHub Actions workflow will [automatically generate and push the site HTML for you](https://github.com/apache/spark-website#readme). After merging the change into the `asf-site` branch, you may need to create a follow-up empty commit to force synchronization between ASF's git and the web site, and also the GitHub mirror. diff --git a/sitemap.xml b/sitemap.xml index 257359ef91c..71fb8b093dc 100644 --- a/sitemap.xml +++ b/sitemap.xml @@ -151,9 +151,18 @@ sitemap: false weekly {% endfor %} -{% for page in site.pages %}{% if page.sitemap != false %} +{%- comment -%} +Explicitly sort `site.pages` so that the order is consistent and we don't get spurious git diffs. +`site.posts` doesn't have this issue because it's already sorted. +See: https://jekyllrb.com/docs/variables/#site-variables +{%- endcomment -%} +{%- assign sorted_pages = site.pages | sort: "url" -%} +{%- for page in sorted_pages -%} +{%- if page.sitemap != false -%} + {{ site.url }}{{ page.url }} weekly -{% endif %} -{% endfor %} + +{% endif %} +{%- endfor -%} diff --git a/third-party-projects.md b/third-party-projects.md index 043af185444..3cadd6ef6a8 100644 --- a/third-party-projects.md +++ b/third-party-projects.md @@ -100,6 +100,6 @@ transforming, and analyzing genomic data using Apache Spark ## Adding new projects -To add a project, open a pull request against the [spark-website](https://github.com/apache/spark-website) repository. Add an entry to [this markdown file](https://github.com/apache/spark-website/blob/asf-site/third-party-projects.md), then run `jekyll build` to generate the HTML too. Include both in your pull request. See the README in this repo for more information. +To add a project, open a pull request against the [spark-website](https://github.com/apache/spark-website) repository. Add an entry to [this markdown file](https://github.com/apache/spark-website/blob/asf-site/third-party-projects.md) and follow the [README instructions](https://github.com/apache/spark-website#readme) to submit a pull request with your changes. Note that all project and product names should follow [trademark guidelines](/trademarks.html).