--- a +++ b/.circleci/config.yml @@ -0,0 +1,559 @@ +# By default, for PRs CircleCI will build only examples that have changed. +# For main commits, builds are skipped entirely, as we only do full builds +# scheduled for one time daily. +# +# Tagging a commit with the following overrides these behaviors: +# - [circle front] will run the front page examples and perform test-doc +# - [circle full] will run all examples and perform test-doc +# - [circle linkcheck] will run our linkcheck job +# - [circle deploy] on a main or maint/* commit will try to immediately build +# and deploy docs rather than waiting for the nightly build + +version: 2.1 + +_check_skip: &check_skip + name: Check-skip + command: | + set -e + export COMMIT_MESSAGE=$(git log --format=oneline -n 1); + if [[ "$CIRCLE_PULL_REQUEST" != "" ]] && ([[ "$COMMIT_MESSAGE" == *"[skip circle]"* ]] || [[ "$COMMIT_MESSAGE" == *"[circle skip]"* ]]); then + echo "Skip detected, exiting job ${CIRCLE_JOB} for PR ${CIRCLE_PULL_REQUEST}." + circleci-agent step halt; + fi + +jobs: + build_docs: + parameters: + scheduled: + type: string + default: "false" + machine: + image: ubuntu-2404:current + # large 4 vCPUs 15GB mem + # https://discuss.circleci.com/t/changes-to-remote-docker-reporting-pricing/47759 + resource_class: large + steps: + - restore_cache: + keys: + - source-cache + - checkout + - run: + name: Complete checkout + command: | + set -e + if ! git remote -v | grep upstream; then + git remote add upstream https://github.com/mne-tools/mne-python.git + fi + git remote set-url upstream https://github.com/mne-tools/mne-python.git + git fetch upstream + - save_cache: + key: source-cache + paths: + - ".git" + - run: + <<: *check_skip + - run: + name: Merge with upstream and triage run + command: | + set -e + echo $(git log -1 --pretty=%B) | tee gitlog.txt + echo ${CI_PULL_REQUEST//*pull\//} | tee merge.txt + if [[ $(cat merge.txt) != "" ]]; then + echo "Merging $(cat merge.txt)"; + git pull --ff-only upstream "refs/pull/$(cat merge.txt)/merge"; + else + if [[ "$CIRCLE_BRANCH" == "main" ]]; then + KIND=dev + else + KIND=stable + fi + export COMMIT_MESSAGE=$(git log --format=oneline -n 1); + if [[ "<< parameters.scheduled >>" == "true" ]]; then + echo "Scheduled full build detected, checking if it's required." + wget https://mne.tools/${KIND}/_version.txt; + REMOTE_VERSION=$(cat _version.txt) + THIS_VERSION=$(git rev-parse HEAD) + echo "Current ${KIND} SHA: ${REMOTE_VERSION}" + echo "This ${KIND} SHA: ${THIS_VERSION}" + if [[ "${THIS_VERSION}" != "${REMOTE_VERSION}" ]]; then + echo "Rebuild required." + else + echo "Rebuild skipped." + circleci-agent step halt; + fi + elif [[ "$COMMIT_MESSAGE" == *"[circle deploy]"* ]]; then + echo "Forced deployed build detected, building and deploying docs"; + else + echo "Waiting until scheduled run to build ${KIND} docs, exiting job ${CIRCLE_JOB}." + circleci-agent step halt; + fi + fi + + - run: + name: Set BASH_ENV + command: ./tools/circleci_bash_env.sh + + - run: + name: check neuromag2ft + command: | + neuromag2ft --version + + - run: + name: Install fonts needed for diagrams + command: | + mkdir -p $HOME/.fonts + echo "Source Code Pro" + curl https://codeload.github.com/adobe-fonts/source-code-pro/tar.gz/2.038R-ro/1.058R-it/1.018R-VAR | tar xz -C $HOME/.fonts + echo "Source Sans Pro" + curl https://codeload.github.com/adobe-fonts/source-sans/tar.gz/3.028R | tar xz -C $HOME/.fonts + fc-cache -f + + # Load pip cache + - restore_cache: + keys: + - pip-cache-0 + - restore_cache: + keys: + - user-install-bin-cache-310 + + # Hack in uninstalls of libraries as necessary if pip doesn't do the right thing in upgrading for us... + - run: + name: Get Python running + command: | + ./tools/circleci_dependencies.sh + + - save_cache: + key: pip-cache-0 + paths: + - ~/.cache/pip + - save_cache: + key: user-install-bin-cache-310 + paths: + - ~/.local/lib/python3.10/site-packages + - ~/.local/bin + + - run: + name: Check Qt + command: | + ./tools/check_qt_import.sh PyQt6 + # Load tiny cache so that ~/.mne does not need to be created below + - restore_cache: + keys: + - data-cache-tiny-0 + + # Look at what we have and fail early if there is some library conflict + - run: + name: Check installation + command: | + which python + QT_DEBUG_PLUGINS=1 mne sys_info -pd + python -c "import numpy; numpy.show_config()" + python -c "import dipy.align.metrics" + LIBGL_DEBUG=verbose python -c "import pyvistaqt; pyvistaqt.BackgroundPlotter(show=True)" + python -c "import mne; mne.set_config('MNE_USE_CUDA', 'false')" # this is needed for the config tutorial + python -c "import mne; mne.set_config('MNE_LOGGING_LEVEL', 'info')" + python -c "import mne; level = mne.get_config('MNE_LOGGING_LEVEL'); assert level.lower() == 'info', repr(level)" + - run: + name: List packages + command: python -m pip list + + # Figure out if we should run a full build or specify a pattern + - restore_cache: + keys: + - data-cache-tiny-1 + - restore_cache: + keys: + - data-cache-multimodal + - restore_cache: + keys: + - data-cache-limo + - restore_cache: + keys: + - data-cache-fsaverage + - restore_cache: + keys: + - data-cache-bst-raw + - restore_cache: + keys: + - data-cache-bst-phantom-ctf + - restore_cache: + keys: + - data-cache-bst-phantom-elekta + - restore_cache: + keys: + - data-cache-bst-phantom-kernel + - restore_cache: + keys: + - data-cache-bst-auditory + - restore_cache: + keys: + - data-cache-bst-resting + - restore_cache: + keys: + - data-cache-fieldtrip + - restore_cache: + keys: + - data-cache-somato + - restore_cache: + keys: + - data-cache-hf-sef + - restore_cache: + keys: + - data-cache-opm + - restore_cache: + keys: + - data-cache-sample + - restore_cache: + keys: + - data-cache-spm-face + - restore_cache: + keys: + - data-cache-testing + - restore_cache: + keys: + - data-cache-visual + - restore_cache: + keys: + - data-cache-ucl-opm-auditory + - restore_cache: + keys: + - data-cache-phantom-kit + - restore_cache: + keys: + - data-cache-ds004388 + - run: + name: Get data + # This limit could be increased, but this is helpful for finding slow ones + # (even ~2GB datasets should be downloadable in this time from good + # providers) + no_output_timeout: 10m + command: | + ./tools/circleci_download.sh + - run: + name: Verify build type + command: | + echo "PATTERN=$(cat pattern.txt)" + echo "BUILD=$(cat build.txt)" + ls -al ~/mne_data; + + # Run doctest (if it's full or front) before building the docs + - run: + name: make test-doc + command: | + if [[ $(cat gitlog.txt) == *"[circle front]"* ]] || [[ $(cat build.txt) == "html-memory" ]] ; then + make test-doc; + mkdir -p doc/_build/test-results/test-doc; + cp junit-results.xml doc/_build/test-results/test-doc/junit.xml; + fi; + # Build docs + - run: + name: make html + command: | # we have -o pipefail in #BASH_ENV so we should be okay + set -x + PATTERN=$(cat pattern.txt) make -C doc $(cat build.txt) 2>&1 | tee sphinx_log.txt + - run: + name: Check sphinx log for warnings (which are treated as errors) + when: always + command: | + ! grep "^.*\(WARNING\|ERROR\): " sphinx_log.txt + - run: + name: Show profiling output + when: always + command: | + if compgen -G "doc/*.dat" > /dev/null; then + mkdir -p doc/generated + mprof plot doc/*.dat --output doc/generated/memory.png + else + echo "No profile data found in doc/" + fi + - run: + name: Sanity check system state + command: | + python -c "import mne; level = mne.get_config('MNE_LOGGING_LEVEL'); assert level.lower() == 'info', repr(level)" + + # Reduce upload time of artifacts we will (almost) never look at + - run: + name: Reduce artifact upload time + command: | + if grep -q html-pattern-memory build.txt; then + zip -rm doc/_build/html/_downloads.zip doc/_build/html/_downloads + fi + for NAME in generated auto_tutorials auto_examples; do + zip -rm doc/${NAME}.zip doc/${NAME} + done + + # Save the JUnit file + - store_test_results: + path: doc/_build/test-results + - store_artifacts: + path: doc/_build/test-results + destination: test-results + # Save the SG RST + - store_artifacts: + path: doc/auto_examples.zip + - store_artifacts: + path: doc/auto_tutorials.zip + - store_artifacts: + path: doc/generated.zip + # Save the HTML + - store_artifacts: + path: doc/_build/html/ + destination: html + - persist_to_workspace: + root: doc/_build + paths: + - html + + # Keep these separate, maybe better in terms of size limitations (?) + - save_cache: + key: data-cache-tiny-0 # < 100 M, might as well combine + paths: + - ~/.mne + - ~/mne_data/MNE-kiloword-data # (28 M) + - ~/mne_data/MNE-eegbci-data # (35 M) + - ~/mne_data/MNE-misc-data # (39 M) + - ~/mne_data/mTRF_1.5 # (56 M) + - ~/mne_data/MNE-phantom-4DBTi # (77 M) + - save_cache: + key: data-cache-tiny-1 # more to combine + paths: + - ~/mne_data/MNE-fNIRS-motor-data # (71 M) + - ~/mne_data/MNE-refmeg-noise-data # (93 M) + - ~/mne_data/physionet-sleep-data # (95 M) + - save_cache: + key: data-cache-multimodal + paths: + - ~/mne_data/MNE-multimodal-data # (240 M) + - save_cache: + key: data-cache-limo + paths: + - ~/mne_data/MNE-limo-data # (244 M) + - save_cache: + key: data-cache-fsaverage + paths: + - ~/mne_data/MNE-fsaverage-data # (762 M) + - save_cache: + key: data-cache-bst-raw + paths: + - ~/mne_data/MNE-brainstorm-data/bst_raw # (830 M) + - save_cache: + key: data-cache-bst-phantom-ctf + paths: + - ~/mne_data/MNE-brainstorm-data/bst_phantom_ctf # (177 M) + - save_cache: + key: data-cache-bst-phantom-elekta + paths: + - ~/mne_data/MNE-brainstorm-data/bst_phantom_elekta # (1.4 G) + - save_cache: + key: data-cache-bst-phantom-kernel + paths: + - ~/mne_data/MNE-phantom-kernel-data # (362 M) + - save_cache: + key: data-cache-bst-auditory + paths: + - ~/mne_data/MNE-brainstorm-data/bst_auditory # (2.9 G) + - save_cache: + key: data-cache-bst-resting + paths: + - ~/mne_data/MNE-brainstorm-data/bst_resting # (4.5 G) + - save_cache: + key: data-cache-fieldtrip + paths: + - ~/mne_data/MNE-fieldtrip_cmc-data # (699 M) + - save_cache: + key: data-cache-somato + paths: + - ~/mne_data/MNE-somato-data # (750 M) + - save_cache: + key: data-cache-hf-sef + paths: + - ~/mne_data/HF_SEF # (1.3 G) + - save_cache: + key: data-cache-opm + paths: + - ~/mne_data/MNE-OPM-data # (1.9 G) + - save_cache: + key: data-cache-sample + paths: + - ~/mne_data/MNE-sample-data # (3.2 G) + - save_cache: + key: data-cache-spm-face + paths: + - ~/mne_data/MNE-spm-face # (1.5 G) + - save_cache: + key: data-cache-testing + paths: + - ~/mne_data/MNE-testing-data # (2.5 G) + - save_cache: + key: data-cache-visual + paths: + - ~/mne_data/MNE-visual_92_categories-data # (6 G) + - save_cache: + key: data-cache-ucl-opm-auditory + paths: + - ~/mne_data/auditory_OPM_stationary # (4 G) + - save_cache: + key: data-cache-phantom-kit + paths: + - ~/mne_data/MNE-phantom-KIT-data # (1 G) + - save_cache: + key: data-cache-ds004388 + paths: + - ~/mne_data/ds004388 # (1.8 G) + + + linkcheck: + # there are a few files excluded from this for expediency, see Makefile + parameters: + scheduled: + type: string + default: "false" + machine: + image: ubuntu-2404:current + resource_class: large + steps: + - restore_cache: + keys: + - source-cache + - checkout + - run: + name: Check-skip + command: | + export COMMIT_MESSAGE=$(git log --format=oneline -n 1); + if [[ "$COMMIT_MESSAGE" != *"[circle linkcheck]"* ]] && [ "<< parameters.scheduled >>" != "true" ]; then + echo "Skip detected, exiting job ${CIRCLE_JOB}." + circleci-agent step halt; + fi + - run: + name: Set BASH_ENV + command: ./tools/circleci_bash_env.sh + - restore_cache: + keys: + - pip-cache-0 + - run: + name: Get Python running + command: | + ./tools/circleci_dependencies.sh + - run: + name: Check installation + command: | + mne sys_info -pd + - run: + name: make linkcheck + no_output_timeout: 40m + command: | + make -C doc linkcheck + - store_artifacts: + path: doc/_build/linkcheck + destination: linkcheck + + + deploy: + machine: + image: ubuntu-2404:current + steps: + - attach_workspace: + at: /tmp/build + - restore_cache: + keys: + - website-cache + - run: + name: Set BASH_ENV + command: | + set -e + echo "set -e" >> $BASH_ENV + # Don't try to deploy if nothing is there or not on the right branch + - run: + name: Check docs + command: | + if [ ! -f /tmp/build/html/index.html ] ; then + echo "No files found to upload (build: ${CIRCLE_BRANCH})."; + circleci-agent step halt; + fi; + - run: + name: Fetch docs + command: | + mkdir -p ~/.ssh + echo -e "Host *\nStrictHostKeyChecking no" > ~/.ssh/config + chmod og= ~/.ssh/config + if [ ! -d ~/mne-tools.github.io ]; then + git clone git@github.com:/mne-tools/mne-tools.github.io.git ~/mne-tools.github.io --depth=1 + fi + - run: + name: Deploy docs + command: | + git config --global user.email "circle@mne.tools"; + git config --global user.name "Circle CI"; + cd ~/mne-tools.github.io; + git checkout main + git remote -v + git fetch origin + git reset --hard origin/main + git clean -xdf + if [ "${CIRCLE_BRANCH}" == "main" ]; then + echo "Deploying dev docs for ${CIRCLE_BRANCH}."; + rm -Rf dev; + cp -a /tmp/build/html dev; + git add -A; + git commit -m "CircleCI update of dev docs (${CIRCLE_BUILD_NUM})."; + else + echo "Deploying stable docs for ${CIRCLE_BRANCH}."; + rm -Rf stable; + cp -a /tmp/build/html stable; + git add -A; + git commit -m "CircleCI update of stable docs (${CIRCLE_BUILD_NUM})."; + fi; + git push origin main; + - save_cache: + key: website-cache + paths: + - ~/mne_data/MNE-visual_92_categories-data + +workflows: + default: + jobs: + - build_docs: + name: build_docs + - linkcheck: + name: linkcheck + - deploy: + name: deploy + requires: + - build_docs + filters: + branches: + only: + - main + - /maint\/.*/ + + main: + jobs: + - build_docs: + scheduled: "true" + name: build_docs_main + - deploy: + name: deploy_main + requires: + - build_docs_main + triggers: + - schedule: + # "At 6:00 AM GMT every day" + cron: "0 6 * * *" + filters: + branches: + only: + - main + + monthly: + jobs: + - linkcheck: + name: linkcheck_monthly + scheduled: "true" + triggers: + - schedule: + # "At 6:00 AM GMT on the first day of each month" + cron: "0 6 1 * *" + filters: + branches: + only: + - main