nightly-tests.yaml

name: nightly-tests

on:
  schedule:
    # Run at midnight UTC
    - cron: "0 0 * * *"
  # Allow manual triggering
  workflow_dispatch:
    inputs:
      shards:
        description: "Comma-separated list of shards to run (leave empty for all)"
        required: false
        default: ""

jobs:
  nightly-tests:
    name: Nightly Tests
    timeout-minutes: 30
    permissions:
      contents: read
      packages: write
    runs-on: ubuntu-latest
    env:
      MAX_ATTEMPTS: 3
    strategy:
      matrix:
        python-version-major: ["3"]
        python-version-minor: [
            "10",
            "11",
            "12",
            "13",
            #
          ]
        shard:
          [
            "trace",
            "flow",
            "trace_server",
            "trace_server_bindings",
            "anthropic",
            "bedrock",
            "cerebras",
            "cohere",
            "crewai",
            "dspy",
            "groq",
            "google_genai",
            "instructor",
            "langchain",
            "litellm",
            "llamaindex",
            "mistral",
            "notdiamond",
            "openai",
            "scorers",
            "verifiers_test",
            "vertexai",
            "pandas_test",
            "fastmcp",
            "smolagents",
            "autogen_tests",
          ]
        exclude:
          # verifiers_test dependencies require Python >= 3.11.
          - python-version-major: "3"
            python-version-minor: "10"
            shard: "verifiers_test"
      fail-fast: false
    steps:
      - name: Checkout
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
      - name: Filter shards if specified
        id: filter_shards
        run: |
          INPUT_SHARDS="${{ github.event.inputs.shards }}"
          if [ -n "$INPUT_SHARDS" ]; then
            echo "Running specific shards: $INPUT_SHARDS"
            if [[ ! ",$INPUT_SHARDS," == *",${{ matrix.shard }},"* ]]; then
              echo "Skipping ${{ matrix.shard }} as it's not in the specified list"
              echo "skip=true" >> $GITHUB_OUTPUT
              exit 0
            fi
          fi
          echo "Running shard: ${{ matrix.shard }}"
          echo "skip=false" >> $GITHUB_OUTPUT
      - name: Start ClickHouse
        if: steps.filter_shards.outputs.skip != 'true'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -euo pipefail
          image=$(.github/scripts/ch-image.sh)
          docker run --detach --rm --name weave_clickhouse \
            --env CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 \
            --env CLICKHOUSE_USER=default \
            --env CLICKHOUSE_PASSWORD= \
            --env CLICKHOUSE_DB=default \
            --publish 8123:8123 \
            -v ${{ github.workspace }}/tests/trace_server/conftest_lib/test_overrides.xml:/etc/clickhouse-server/users.d/test_overrides.xml:ro \
            "$image"
          for _i in {1..30}; do
            if wget -q -O- 'http://localhost:8123/ping' >/dev/null; then
              echo "clickhouse is healthy"
              exit 0
            fi
            sleep 1
          done
          docker logs weave_clickhouse
          exit 1
      - name: Enable debug logging
        if: steps.filter_shards.outputs.skip != 'true'
        run: echo "ACTIONS_STEP_DEBUG=true" >> $GITHUB_ENV
      - name: Set up Python ${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}
        if: steps.filter_shards.outputs.skip != 'true'
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
        with:
          python-version: ${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}
      - name: Install dependencies
        if: steps.filter_shards.outputs.skip != 'true'
        run: |
          python -m pip install --upgrade pip
          pip install nox uv
      - name: Check if shard needs trace_server markers
        id: check_trace_markers
        if: steps.filter_shards.outputs.skip != 'true'
        run: |
          # Shards that are in the regular test matrix and need trace_server markers
          TRACE_SERVER_SHARDS="flow trace_server trace_server_bindings anthropic cerebras cohere crewai dspy groq google_genai google_ai_studio instructor langchain litellm llamaindex mistral notdiamond openai openai_agents vertexai bedrock scorers pandas_test huggingface smolagents fastmcp autogen_tests trace trace_no_server"
          if [[ " $TRACE_SERVER_SHARDS " =~ " ${{ matrix.shard }} " ]]; then
            echo "needs_markers=true" >> $GITHUB_OUTPUT
          else
            echo "needs_markers=false" >> $GITHUB_OUTPUT
          fi
      - name: Run nox (Clickhouse Trace Server)
        if: steps.filter_shards.outputs.skip != 'true' && steps.check_trace_markers.outputs.needs_markers == 'true'
        env:
          WEAVE_SENTRY_ENV: ci
          CI: 1
          WEAVE_USE_MEMRAY: 1
          WF_CLICKHOUSE_HOST: localhost
          WEAVE_SERVER_DISABLE_ECOSYSTEM: 1
          WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          DD_TRACE_ENABLED: false
        run: |
          .github/scripts/retry.sh nox -e "tests-${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}(shard='${{ matrix.shard }}')" -- \
            -m "trace_server"
      - name: Run nox (No marker filters for integration-only shards)
        if: steps.filter_shards.outputs.skip != 'true' && steps.check_trace_markers.outputs.needs_markers == 'false'
        env:
          WEAVE_SENTRY_ENV: ci
          CI: 1
          WEAVE_USE_MEMRAY: 1
          WF_CLICKHOUSE_HOST: localhost
          WEAVE_SERVER_DISABLE_ECOSYSTEM: 1
          WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          DD_TRACE_ENABLED: false
        run: |
          .github/scripts/retry.sh nox -e "tests-${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}(shard='${{ matrix.shard }}')"

  nightly-tests-replicated-1s3r:
    name: Nightly Tests (replicated 1s3r)
    permissions:
      contents: read
    uses: ./.github/workflows/nightly-replicated-leg.yaml
    with:
      topology: "1s3r"
      ch_port: "8130"
      use_distributed: false
      shards: ${{ github.event.inputs.shards }}
    secrets: inherit

  nightly-tests-replicated-2s2r:
    name: Nightly Tests (replicated 2s2r)
    permissions:
      contents: read
    uses: ./.github/workflows/nightly-replicated-leg.yaml
    with:
      topology: "2s2r"
      ch_port: "8131"
      use_distributed: true
      shards: ${{ github.event.inputs.shards }}
    secrets: inherit

  slack-notification:
    name: Slack Notification
    needs:
      - nightly-tests
      - nightly-tests-replicated-1s3r
      - nightly-tests-replicated-2s2r

    runs-on: ubuntu-latest
    if: always()

    steps:
      - name: Determine Result and Failed Jobs
        id: result
        env:
          NEEDS: "${{ toJson(needs) }}"
        run: |
          jq -C . <<< "${NEEDS}" # debug

          # Check if any job failed
          has_failure=$(
            jq -r '
              .[]
              | select(.result == "failure")
              | "true"
            ' <<< "${NEEDS}" | head -n1
          )

          if [ "$has_failure" = "true" ]; then
            echo "status=failure" >> $GITHUB_OUTPUT
            
            failed_jobs=$(
              jq -r '
                to_entries
                | [
                  .[]
                  | select(.value.result == "failure")
                  | .key
                ]
                | join(", ")
              ' <<< "${NEEDS}"
            )
            echo "failed_jobs=$failed_jobs" >> $GITHUB_OUTPUT
          else
            echo "status=success" >> $GITHUB_OUTPUT
          fi

      - name: Send Slack Notification on Failure
        if: steps.result.outputs.status == 'failure'
        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
        with:
          channel-id: "C09CE03J867"
          slack-message: |
            <!subteam^S06HYJNN7SL> :x: Nightly Tests failed!
            *Repository:* ${{ github.repository }}
            *Commit:* ${{ github.sha }}
            *Failed Jobs:* ${{ steps.result.outputs.failed_jobs }}
            *Build:* ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        env:
          SLACK_BOT_TOKEN: ${{ secrets.WANDB_CI_SLACK_BOT_TOKEN }}

      - name: Send Slack Notification on Success
        if: steps.result.outputs.status == 'success'
        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
        with:
          channel-id: "C09CE03J867"
          slack-message: |
            :white_check_mark: Nightly Tests passed!
            *Repository:* ${{ github.repository }}
            *Commit:* ${{ github.sha }}
            *Build:* ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        env:
          SLACK_BOT_TOKEN: ${{ secrets.WANDB_CI_SLACK_BOT_TOKEN }}