Diff of /.circleci/config.yml [000000] .. [7f9fb8]

Switch to side-by-side view

--- a
+++ b/.circleci/config.yml
@@ -0,0 +1,559 @@
+# By default, for PRs CircleCI will build only examples that have changed.
+# For main commits, builds are skipped entirely, as we only do full builds
+# scheduled for one time daily.
+#
+# Tagging a commit with the following overrides these behaviors:
+# - [circle front] will run the front page examples and perform test-doc
+# - [circle full] will run all examples and perform test-doc
+# - [circle linkcheck] will run our linkcheck job
+# - [circle deploy] on a main or maint/* commit will try to immediately build
+#   and deploy docs rather than waiting for the nightly build
+
+version: 2.1
+
+_check_skip: &check_skip
+  name: Check-skip
+  command: |
+    set -e
+    export COMMIT_MESSAGE=$(git log --format=oneline -n 1);
+    if [[ "$CIRCLE_PULL_REQUEST" != "" ]] && ([[ "$COMMIT_MESSAGE" == *"[skip circle]"* ]] || [[ "$COMMIT_MESSAGE" == *"[circle skip]"* ]]); then
+      echo "Skip detected, exiting job ${CIRCLE_JOB} for PR ${CIRCLE_PULL_REQUEST}."
+      circleci-agent step halt;
+    fi
+
+jobs:
+  build_docs:
+    parameters:
+      scheduled:
+        type: string
+        default: "false"
+    machine:
+      image: ubuntu-2404:current
+    # large 4 vCPUs 15GB mem
+    # https://discuss.circleci.com/t/changes-to-remote-docker-reporting-pricing/47759
+    resource_class: large
+    steps:
+      - restore_cache:
+          keys:
+            - source-cache
+      - checkout
+      - run:
+          name: Complete checkout
+          command: |
+            set -e
+            if ! git remote -v | grep upstream; then
+              git remote add upstream https://github.com/mne-tools/mne-python.git
+            fi
+            git remote set-url upstream https://github.com/mne-tools/mne-python.git
+            git fetch upstream
+      - save_cache:
+          key: source-cache
+          paths:
+            - ".git"
+      - run:
+          <<: *check_skip
+      - run:
+          name: Merge with upstream and triage run
+          command: |
+            set -e
+            echo $(git log -1 --pretty=%B) | tee gitlog.txt
+            echo ${CI_PULL_REQUEST//*pull\//} | tee merge.txt
+            if [[ $(cat merge.txt) != "" ]]; then
+              echo "Merging $(cat merge.txt)";
+              git pull --ff-only upstream "refs/pull/$(cat merge.txt)/merge";
+            else
+              if [[ "$CIRCLE_BRANCH" == "main" ]]; then
+                KIND=dev
+              else
+                KIND=stable
+              fi
+              export COMMIT_MESSAGE=$(git log --format=oneline -n 1);
+              if [[ "<< parameters.scheduled >>" == "true" ]]; then
+                echo "Scheduled full build detected, checking if it's required."
+                wget https://mne.tools/${KIND}/_version.txt;
+                REMOTE_VERSION=$(cat _version.txt)
+                THIS_VERSION=$(git rev-parse HEAD)
+                echo "Current ${KIND} SHA: ${REMOTE_VERSION}"
+                echo "This    ${KIND} SHA: ${THIS_VERSION}"
+                if [[ "${THIS_VERSION}" != "${REMOTE_VERSION}" ]]; then
+                  echo "Rebuild required."
+                else
+                  echo "Rebuild skipped."
+                  circleci-agent step halt;
+                fi
+              elif [[ "$COMMIT_MESSAGE" == *"[circle deploy]"* ]]; then
+                echo "Forced deployed build detected, building and deploying docs";
+              else
+                echo "Waiting until scheduled run to build ${KIND} docs, exiting job ${CIRCLE_JOB}."
+                circleci-agent step halt;
+              fi
+            fi
+
+      - run:
+          name: Set BASH_ENV
+          command: ./tools/circleci_bash_env.sh
+
+      - run:
+          name: check neuromag2ft
+          command: |
+            neuromag2ft --version
+
+      - run:
+          name: Install fonts needed for diagrams
+          command: |
+            mkdir -p $HOME/.fonts
+            echo "Source Code Pro"
+            curl https://codeload.github.com/adobe-fonts/source-code-pro/tar.gz/2.038R-ro/1.058R-it/1.018R-VAR | tar xz -C $HOME/.fonts
+            echo "Source Sans Pro"
+            curl https://codeload.github.com/adobe-fonts/source-sans/tar.gz/3.028R | tar xz -C $HOME/.fonts
+            fc-cache -f
+
+      # Load pip cache
+      - restore_cache:
+          keys:
+            - pip-cache-0
+      - restore_cache:
+          keys:
+            - user-install-bin-cache-310
+
+      # Hack in uninstalls of libraries as necessary if pip doesn't do the right thing in upgrading for us...
+      - run:
+          name: Get Python running
+          command: |
+            ./tools/circleci_dependencies.sh
+
+      - save_cache:
+          key: pip-cache-0
+          paths:
+            - ~/.cache/pip
+      - save_cache:
+          key: user-install-bin-cache-310
+          paths:
+            - ~/.local/lib/python3.10/site-packages
+            - ~/.local/bin
+
+      - run:
+          name: Check Qt
+          command: |
+            ./tools/check_qt_import.sh PyQt6
+      # Load tiny cache so that ~/.mne does not need to be created below
+      - restore_cache:
+          keys:
+            - data-cache-tiny-0
+
+      # Look at what we have and fail early if there is some library conflict
+      - run:
+          name: Check installation
+          command: |
+              which python
+              QT_DEBUG_PLUGINS=1 mne sys_info -pd
+              python -c "import numpy; numpy.show_config()"
+              python -c "import dipy.align.metrics"
+              LIBGL_DEBUG=verbose python -c "import pyvistaqt; pyvistaqt.BackgroundPlotter(show=True)"
+              python -c "import mne; mne.set_config('MNE_USE_CUDA', 'false')"  # this is needed for the config tutorial
+              python -c "import mne; mne.set_config('MNE_LOGGING_LEVEL', 'info')"
+              python -c "import mne; level = mne.get_config('MNE_LOGGING_LEVEL'); assert level.lower() == 'info', repr(level)"
+      - run:
+          name: List packages
+          command: python -m pip list
+
+      # Figure out if we should run a full build or specify a pattern
+      - restore_cache:
+          keys:
+            - data-cache-tiny-1
+      - restore_cache:
+          keys:
+            - data-cache-multimodal
+      - restore_cache:
+          keys:
+            - data-cache-limo
+      - restore_cache:
+          keys:
+            - data-cache-fsaverage
+      - restore_cache:
+          keys:
+            - data-cache-bst-raw
+      - restore_cache:
+          keys:
+            - data-cache-bst-phantom-ctf
+      - restore_cache:
+          keys:
+            - data-cache-bst-phantom-elekta
+      - restore_cache:
+          keys:
+            - data-cache-bst-phantom-kernel
+      - restore_cache:
+          keys:
+            - data-cache-bst-auditory
+      - restore_cache:
+          keys:
+            - data-cache-bst-resting
+      - restore_cache:
+          keys:
+            - data-cache-fieldtrip
+      - restore_cache:
+          keys:
+            - data-cache-somato
+      - restore_cache:
+          keys:
+            - data-cache-hf-sef
+      - restore_cache:
+          keys:
+            - data-cache-opm
+      - restore_cache:
+          keys:
+            - data-cache-sample
+      - restore_cache:
+          keys:
+            - data-cache-spm-face
+      - restore_cache:
+          keys:
+            - data-cache-testing
+      - restore_cache:
+          keys:
+            - data-cache-visual
+      - restore_cache:
+          keys:
+            - data-cache-ucl-opm-auditory
+      - restore_cache:
+          keys:
+            - data-cache-phantom-kit
+      - restore_cache:
+          keys:
+            - data-cache-ds004388
+      - run:
+          name: Get data
+          # This limit could be increased, but this is helpful for finding slow ones
+          # (even ~2GB datasets should be downloadable in this time from good
+          # providers)
+          no_output_timeout: 10m
+          command: |
+            ./tools/circleci_download.sh
+      - run:
+          name: Verify build type
+          command: |
+            echo "PATTERN=$(cat pattern.txt)"
+            echo "BUILD=$(cat build.txt)"
+            ls -al ~/mne_data;
+
+      # Run doctest (if it's full or front) before building the docs
+      - run:
+          name: make test-doc
+          command: |
+            if [[ $(cat gitlog.txt) == *"[circle front]"* ]] || [[ $(cat build.txt) == "html-memory" ]] ; then
+              make test-doc;
+              mkdir -p doc/_build/test-results/test-doc;
+              cp junit-results.xml doc/_build/test-results/test-doc/junit.xml;
+            fi;
+      # Build docs
+      - run:
+          name: make html
+          command: |  # we have -o pipefail in #BASH_ENV so we should be okay
+            set -x
+            PATTERN=$(cat pattern.txt) make -C doc $(cat build.txt) 2>&1 | tee sphinx_log.txt
+      - run:
+          name: Check sphinx log for warnings (which are treated as errors)
+          when: always
+          command: |
+            ! grep "^.*\(WARNING\|ERROR\): " sphinx_log.txt
+      - run:
+          name: Show profiling output
+          when: always
+          command: |
+            if compgen -G "doc/*.dat" > /dev/null; then
+              mkdir -p doc/generated
+              mprof plot doc/*.dat --output doc/generated/memory.png
+            else
+              echo "No profile data found in doc/"
+            fi
+      - run:
+          name: Sanity check system state
+          command: |
+            python -c "import mne; level = mne.get_config('MNE_LOGGING_LEVEL'); assert level.lower() == 'info', repr(level)"
+
+      # Reduce upload time of artifacts we will (almost) never look at
+      - run:
+          name: Reduce artifact upload time
+          command: |
+            if grep -q html-pattern-memory build.txt; then
+              zip -rm doc/_build/html/_downloads.zip doc/_build/html/_downloads
+            fi
+            for NAME in generated auto_tutorials auto_examples; do
+              zip -rm doc/${NAME}.zip doc/${NAME}
+            done
+
+      # Save the JUnit file
+      - store_test_results:
+          path: doc/_build/test-results
+      - store_artifacts:
+          path: doc/_build/test-results
+          destination: test-results
+      # Save the SG RST
+      - store_artifacts:
+          path: doc/auto_examples.zip
+      - store_artifacts:
+          path: doc/auto_tutorials.zip
+      - store_artifacts:
+          path: doc/generated.zip
+      # Save the HTML
+      - store_artifacts:
+          path: doc/_build/html/
+          destination: html
+      - persist_to_workspace:
+          root: doc/_build
+          paths:
+            - html
+
+      # Keep these separate, maybe better in terms of size limitations (?)
+      - save_cache:
+          key: data-cache-tiny-0  # < 100 M, might as well combine
+          paths:
+            - ~/.mne
+            - ~/mne_data/MNE-kiloword-data  # (28 M)
+            - ~/mne_data/MNE-eegbci-data  # (35 M)
+            - ~/mne_data/MNE-misc-data  # (39 M)
+            - ~/mne_data/mTRF_1.5  # (56 M)
+            - ~/mne_data/MNE-phantom-4DBTi  # (77 M)
+      - save_cache:
+          key: data-cache-tiny-1  # more to combine
+          paths:
+            - ~/mne_data/MNE-fNIRS-motor-data  # (71 M)
+            - ~/mne_data/MNE-refmeg-noise-data  # (93 M)
+            - ~/mne_data/physionet-sleep-data  # (95 M)
+      - save_cache:
+          key: data-cache-multimodal
+          paths:
+            - ~/mne_data/MNE-multimodal-data  # (240 M)
+      - save_cache:
+          key: data-cache-limo
+          paths:
+            - ~/mne_data/MNE-limo-data  # (244 M)
+      - save_cache:
+          key: data-cache-fsaverage
+          paths:
+            - ~/mne_data/MNE-fsaverage-data  # (762 M)
+      - save_cache:
+          key: data-cache-bst-raw
+          paths:
+            - ~/mne_data/MNE-brainstorm-data/bst_raw  # (830 M)
+      - save_cache:
+          key: data-cache-bst-phantom-ctf
+          paths:
+            - ~/mne_data/MNE-brainstorm-data/bst_phantom_ctf  # (177 M)
+      - save_cache:
+          key: data-cache-bst-phantom-elekta
+          paths:
+            - ~/mne_data/MNE-brainstorm-data/bst_phantom_elekta  # (1.4 G)
+      - save_cache:
+          key: data-cache-bst-phantom-kernel
+          paths:
+            - ~/mne_data/MNE-phantom-kernel-data  # (362 M)
+      - save_cache:
+          key: data-cache-bst-auditory
+          paths:
+            - ~/mne_data/MNE-brainstorm-data/bst_auditory  # (2.9 G)
+      - save_cache:
+          key: data-cache-bst-resting
+          paths:
+            - ~/mne_data/MNE-brainstorm-data/bst_resting  # (4.5 G)
+      - save_cache:
+          key: data-cache-fieldtrip
+          paths:
+            - ~/mne_data/MNE-fieldtrip_cmc-data  # (699 M)
+      - save_cache:
+          key: data-cache-somato
+          paths:
+            - ~/mne_data/MNE-somato-data  # (750 M)
+      - save_cache:
+          key: data-cache-hf-sef
+          paths:
+            - ~/mne_data/HF_SEF  # (1.3 G)
+      - save_cache:
+          key: data-cache-opm
+          paths:
+            - ~/mne_data/MNE-OPM-data  # (1.9 G)
+      - save_cache:
+          key: data-cache-sample
+          paths:
+            - ~/mne_data/MNE-sample-data  # (3.2 G)
+      - save_cache:
+          key: data-cache-spm-face
+          paths:
+            - ~/mne_data/MNE-spm-face  # (1.5 G)
+      - save_cache:
+          key: data-cache-testing
+          paths:
+            - ~/mne_data/MNE-testing-data  # (2.5 G)
+      - save_cache:
+          key: data-cache-visual
+          paths:
+            - ~/mne_data/MNE-visual_92_categories-data  # (6 G)
+      - save_cache:
+          key: data-cache-ucl-opm-auditory
+          paths:
+            - ~/mne_data/auditory_OPM_stationary  # (4 G)
+      - save_cache:
+          key: data-cache-phantom-kit
+          paths:
+            - ~/mne_data/MNE-phantom-KIT-data  # (1 G)
+      - save_cache:
+          key: data-cache-ds004388
+          paths:
+            - ~/mne_data/ds004388  # (1.8 G)
+
+
+  linkcheck:
+    # there are a few files excluded from this for expediency, see Makefile
+    parameters:
+      scheduled:
+        type: string
+        default: "false"
+    machine:
+      image: ubuntu-2404:current
+    resource_class: large
+    steps:
+      - restore_cache:
+          keys:
+            - source-cache
+      - checkout
+      - run:
+          name: Check-skip
+          command: |
+            export COMMIT_MESSAGE=$(git log --format=oneline -n 1);
+            if [[ "$COMMIT_MESSAGE" != *"[circle linkcheck]"* ]] && [ "<< parameters.scheduled >>" != "true" ]; then
+              echo "Skip detected, exiting job ${CIRCLE_JOB}."
+              circleci-agent step halt;
+            fi
+      - run:
+          name: Set BASH_ENV
+          command: ./tools/circleci_bash_env.sh
+      - restore_cache:
+          keys:
+            - pip-cache-0
+      - run:
+          name: Get Python running
+          command: |
+            ./tools/circleci_dependencies.sh
+      - run:
+          name: Check installation
+          command: |
+            mne sys_info -pd
+      - run:
+          name: make linkcheck
+          no_output_timeout: 40m
+          command: |
+            make -C doc linkcheck
+      - store_artifacts:
+          path: doc/_build/linkcheck
+          destination: linkcheck
+
+
+  deploy:
+    machine:
+      image: ubuntu-2404:current
+    steps:
+      - attach_workspace:
+          at: /tmp/build
+      - restore_cache:
+          keys:
+            - website-cache
+      - run:
+          name: Set BASH_ENV
+          command: |
+            set -e
+            echo "set -e" >> $BASH_ENV
+      # Don't try to deploy if nothing is there or not on the right branch
+      - run:
+          name: Check docs
+          command: |
+            if [ ! -f /tmp/build/html/index.html ] ; then
+              echo "No files found to upload (build: ${CIRCLE_BRANCH}).";
+              circleci-agent step halt;
+            fi;
+      - run:
+          name: Fetch docs
+          command: |
+            mkdir -p ~/.ssh
+            echo -e "Host *\nStrictHostKeyChecking no" > ~/.ssh/config
+            chmod og= ~/.ssh/config
+            if [ ! -d ~/mne-tools.github.io ]; then
+              git clone git@github.com:/mne-tools/mne-tools.github.io.git ~/mne-tools.github.io --depth=1
+            fi
+      - run:
+          name: Deploy docs
+          command: |
+            git config --global user.email "circle@mne.tools";
+            git config --global user.name "Circle CI";
+            cd ~/mne-tools.github.io;
+            git checkout main
+            git remote -v
+            git fetch origin
+            git reset --hard origin/main
+            git clean -xdf
+            if [ "${CIRCLE_BRANCH}" == "main" ]; then
+              echo "Deploying dev docs for ${CIRCLE_BRANCH}.";
+              rm -Rf dev;
+              cp -a /tmp/build/html dev;
+              git add -A;
+              git commit -m "CircleCI update of dev docs (${CIRCLE_BUILD_NUM}).";
+            else
+              echo "Deploying stable docs for ${CIRCLE_BRANCH}.";
+              rm -Rf stable;
+              cp -a /tmp/build/html stable;
+              git add -A;
+              git commit -m "CircleCI update of stable docs (${CIRCLE_BUILD_NUM}).";
+            fi;
+            git push origin main;
+      - save_cache:
+          key: website-cache
+          paths:
+            - ~/mne_data/MNE-visual_92_categories-data
+
+workflows:
+  default:
+    jobs:
+      - build_docs:
+          name: build_docs
+      - linkcheck:
+          name: linkcheck
+      - deploy:
+          name: deploy
+          requires:
+            - build_docs
+          filters:
+            branches:
+              only:
+                - main
+                - /maint\/.*/
+
+  main:
+    jobs:
+      - build_docs:
+          scheduled: "true"
+          name: build_docs_main
+      - deploy:
+          name: deploy_main
+          requires:
+            - build_docs_main
+    triggers:
+      - schedule:
+          # "At 6:00 AM GMT every day"
+          cron: "0 6 * * *"
+          filters:
+            branches:
+              only:
+                - main
+
+  monthly:
+    jobs:
+      - linkcheck:
+          name: linkcheck_monthly
+          scheduled: "true"
+    triggers:
+      - schedule:
+          # "At 6:00 AM GMT on the first day of each month"
+          cron: "0 6 1 * *"
+          filters:
+            branches:
+              only:
+                - main