Skip to content

fix(weave): annotate _prepare_feedback_create #52323

fix(weave): annotate _prepare_feedback_create

fix(weave): annotate _prepare_feedback_create #52323

Workflow file for this run

# We use self-hosted runners for performance: we can rely on docker caching
# on the runner machines for 0s startup times for our tests.
# There is an issue with self-hosted runners where if you use a mix of
# host and container jobs, the host jobs will fail to start
# (https://github.com/actions/checkout/issues/273).
# So we create a separate "builder" tagged runner to do the build job on
# the host, and then all the tests run on "runner" tagged runners.
name: test
on:
push:
jobs:
check-which-tests-to-run:
uses: ./.github/workflows/check-which-tests-to-run.yaml
weave-ts-sdk-lint:
name: Weave TS SDK Lint
runs-on: ubuntu-latest
needs:
- check-which-tests-to-run
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
with:
fetch-depth: 0
- name: Check if lint should run
id: check_run
run: |
if [[ "${{ github.ref }}" == "refs/heads/master" || "${{ needs.check-which-tests-to-run.outputs.weave_ts_sdk_tests }}" == "true" ]]; then
echo "Lint should run"
echo "should_lint=true" >> $GITHUB_OUTPUT
else
echo "Lint should not run"
echo "should_lint=false" >> $GITHUB_OUTPUT
fi
- uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
with:
version: 10.8.1
- name: Run Weave TS SDK Lint
if: steps.check_run.outputs.should_lint == 'true'
run: |
cd sdks/node
pnpm install --frozen-lockfile
pnpm run prettier-check
# ==== Trace Jobs ====
lint:
name: Python lint
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
with:
python-version: "3.10"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install nox uv
- name: Run nox
run: nox -e lint
trace_no_server:
name: Trace non-trace server tests
timeout-minutes: 10
runs-on: ubuntu-latest
strategy:
matrix:
python-version-major: ["3"]
python-version-minor: [
"10",
"11",
"13",
#
]
nox-shard: ["trace_no_server"]
fail-fast: false
steps:
- name: Checkout
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
- name: Set up Python ${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
with:
python-version: ${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install nox uv
- name: Run nox (Non Trace Server)
run: |
nox -e "tests-${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}(shard='${{ matrix.nox-shard }}')" -- -m "not trace_server"
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5
with:
files: ./coverage.xml
flags: trace-no-server
name: trace-no-server-${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}
fail_ci_if_error: false
token: ${{ secrets.CODECOV_TOKEN }}
trace-tests:
name: Trace nox tests
timeout-minutes: 15
permissions:
contents: read
packages: write
# TODO: refactor to use matrix.include for runner selection
# Use 8-core runners for trace and trace_calls_merged_only shards (resource-intensive)
runs-on: ${{ (matrix.nox-shard == 'trace' || matrix.nox-shard == 'trace_calls_merged_only') && 'ubuntu-latest-8-cores' || 'ubuntu-latest' }}
strategy:
matrix:
python-version-major: ["3"]
python-version-minor: [
"10",
"11",
"13",
#
]
nox-shard: [
"trace",
"trace_calls_merged_only",
"flow",
"trace_server",
"trace_server_bindings",
"anthropic",
"bedrock",
"cerebras",
"cohere",
"crewai",
"dspy",
"groq",
"google_genai",
"google_adk",
"instructor",
"langchain",
"litellm",
"llamaindex",
"mistral",
"notdiamond",
"openai",
"openai_agents",
"scorers",
"verifiers_test",
"vertexai",
"pandas_test",
"fastmcp",
"smolagents",
"stainless",
"autogen_tests",
]
# The Claude Agent SDK has both ClickHouse-backed calls tests and
# no-server OTel tests. Run it once to collect coverage for both paths
# without multiplying the integration shard across every Python version.
include:
- python-version-major: "3"
python-version-minor: "10"
nox-shard: "claude_agent_sdk"
# Shards that don't support certain Python versions. Excluding here
# (vs. skipping inside noxfile) avoids spinning up a runner just to
# emit an empty coverage.xml that Codecov then rejects.
exclude:
- python-version-minor: "10"
nox-shard: "notdiamond"
- python-version-minor: "10"
nox-shard: "verifiers_test"
- python-version-minor: "13"
nox-shard: "cohere"
- python-version-minor: "13"
nox-shard: "notdiamond"
- python-version-minor: "13"
nox-shard: "verifiers_test"
fail-fast: false
steps:
- name: Checkout
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
- name: Start ClickHouse
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
image=$(.github/scripts/ch-image.sh)
docker run --detach --rm --name weave_clickhouse \
--env CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 \
--env CLICKHOUSE_USER=default \
--env CLICKHOUSE_PASSWORD= \
--env CLICKHOUSE_DB=default \
--publish 8123:8123 \
-v ${{ github.workspace }}/tests/trace_server/conftest_lib/test_overrides.xml:/etc/clickhouse-server/users.d/test_overrides.xml:ro \
"$image"
for _i in {1..30}; do
if wget -q -O- 'http://localhost:8123/ping' >/dev/null; then
echo "clickhouse is healthy"
exit 0
fi
sleep 1
done
docker logs weave_clickhouse
exit 1
- name: Enable debug logging
run: echo "ACTIONS_STEP_DEBUG=true" >> $GITHUB_ENV
- name: Set up Python ${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
with:
python-version: ${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install nox uv
- name: Run nox (Clickhouse Trace Server)
env:
WEAVE_SENTRY_ENV: ci
CI: 1
WF_CLICKHOUSE_HOST: localhost
WEAVE_SERVER_DISABLE_ECOSYSTEM: 1
# Use format-valid dummy API keys when secrets aren't available (e.g., Dependabot PRs)
# Tests use VCR recordings so real keys aren't needed
WANDB_API_KEY: ${{ secrets.WANDB_API_KEY || 'DUMMY_API_KEY' }}
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY || 'DUMMY_API_KEY' }}
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY || 'DUMMY_API_KEY' }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY || 'DUMMY_API_KEY' }}
MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY || 'DUMMY_API_KEY' }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY || 'sk-1234567890abcdef1234567890abcdef' }}
DD_TRACE_ENABLED: false
run: |
if [[ "${{ matrix.nox-shard }}" == "claude_agent_sdk" ]]; then
nox -e "tests-${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}(shard='${{ matrix.nox-shard }}')"
else
nox -e "tests-${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}(shard='${{ matrix.nox-shard }}')" -- \
-m "trace_server"
fi
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5
with:
files: ./coverage.xml
flags: trace-tests
name: trace-tests-${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}-${{ matrix.nox-shard }}
fail_ci_if_error: false
token: ${{ secrets.CODECOV_TOKEN }}
fake-trace-tests:
name: Fake (in-memory) trace server tests
timeout-minutes: 15
permissions:
contents: read
# The in-memory fake needs no ClickHouse/wandb services — it is a pure
# in-process backend. Tests it cannot satisfy yet are skipped via
# per-test @pytest.mark.skipif(FAKE_NOT_IMPLEMENTED, ...) decorators
# (see tests/trace/util.py); the rest must match ClickHouse.
runs-on: ubuntu-latest
strategy:
matrix:
python-version-major: ["3"]
python-version-minor: ["13"]
nox-shard: ["trace", "trace_server"]
fail-fast: false
steps:
- name: Checkout
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
- name: Set up Python ${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
with:
python-version: ${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install nox uv
- name: Run nox (Fake trace server)
env:
WEAVE_SENTRY_ENV: ci
CI: 1
WEAVE_SERVER_DISABLE_ECOSYSTEM: 1
DD_TRACE_ENABLED: false
WANDB_API_KEY: ${{ secrets.WANDB_API_KEY || 'DUMMY_API_KEY' }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY || 'sk-1234567890abcdef1234567890abcdef' }}
run: |
nox -e "tests-${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}(shard='${{ matrix.nox-shard }}')" -- \
-m "trace_server" --trace-server=fake
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5
with:
files: ./coverage.xml
flags: fake-trace-tests
name: fake-trace-tests-${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}-${{ matrix.nox-shard }}
fail_ci_if_error: false
token: ${{ secrets.CODECOV_TOKEN }}
trace-server-migrator-1s1r:
name: Trace server migrator tests (1s1r)
timeout-minutes: 10
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
strategy:
matrix:
python-version-major: ["3"]
python-version-minor: ["12"]
fail-fast: false
steps:
- name: Checkout
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
- name: Start ClickHouse with embedded Keeper
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
image=$(.github/scripts/ch-image.sh)
docker run -d --rm \
-e CLICKHOUSE_DB=default \
-e CLICKHOUSE_USER=default \
-e "CLICKHOUSE_PASSWORD=" \
-e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 \
-p 8130:8123 \
-v ${{ github.workspace }}/tests/trace_server/conftest_lib/clickhouse_keeper_config.xml:/etc/clickhouse-server/config.d/keeper.xml \
-v ${{ github.workspace }}/tests/trace_server/conftest_lib/test_overrides.xml:/etc/clickhouse-server/users.d/test_overrides.xml:ro \
--name weave-test-keeper \
--ulimit nofile=262144:262144 \
"$image"
# Wait for server to be healthy
for i in $(seq 1 30); do
wget -q -O- 'http://localhost:8130/ping' && break || sleep 1
done
- name: Set up Python ${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
with:
python-version: ${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install nox uv
- name: Run migrator functional tests
env:
CI: 1
WF_CLICKHOUSE_KEEPER_PORT: 8130
DD_TRACE_ENABLED: false
run: |
nox -e "tests-${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}(shard='trace_server_migrator')"
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5
with:
files: ./coverage.xml
flags: trace-server-migrator-1s1r
name: trace-server-migrator-1s1r-${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}
fail_ci_if_error: false
token: ${{ secrets.CODECOV_TOKEN }}
trace-server-migrator-1s3r:
name: Trace server migrator tests (1s3r)
needs:
- check-which-tests-to-run
if: needs.check-which-tests-to-run.outputs.trace_server_migrator_tests == 'true' || github.ref == 'refs/heads/master'
timeout-minutes: 15
runs-on: ubuntu-latest
strategy:
matrix:
python-version-major: ["3"]
python-version-minor: ["12"]
fail-fast: false
steps:
- name: Checkout
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
- name: Start 1s3r ClickHouse cluster
working-directory: tests/trace_server/conftest_lib/topologies
run: |
docker compose -f 1s3r.yaml up -d --wait
# Extra belt-and-suspenders health probe from host perspective.
for i in $(seq 1 60); do
wget -q -O- 'http://localhost:8130/ping' && break || sleep 2
done
# Show cluster membership so failures are diagnosable from log.
docker compose -f 1s3r.yaml exec -T ch-s1-r1 \
clickhouse-client -q "SELECT host_name, shard_num, replica_num, errors_count FROM system.clusters WHERE cluster='weave_cluster' FORMAT PrettyCompact"
- name: Set up Python ${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
with:
python-version: ${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install nox uv
- name: Run migrator functional tests
env:
CI: 1
# Existing fixture reads WF_CLICKHOUSE_KEEPER_PORT and connects
# to localhost at that port — we map the 1s3r entrypoint to 8130.
# Other WF_CLICKHOUSE_* vars are intentionally NOT set: the
# existing migrator tests pass replicated/distributed flags
# explicitly to the migrator factory, so env vars would be
# unread documentation.
WF_CLICKHOUSE_KEEPER_PORT: 8130
DD_TRACE_ENABLED: false
run: |
nox -e "tests-${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}(shard='trace_server_migrator')"
- name: Capture ClickHouse logs on failure
if: failure()
working-directory: tests/trace_server/conftest_lib/topologies
run: |
docker compose -f 1s3r.yaml logs --no-color --tail=1000 > ch-logs-1s3r.txt || true
- name: Upload ClickHouse logs artifact
if: failure()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: ch-logs-1s3r-${{ github.run_id }}-${{ github.run_attempt }}
path: tests/trace_server/conftest_lib/topologies/ch-logs-1s3r.txt
retention-days: 7
if-no-files-found: ignore
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5
with:
files: ./coverage.xml
flags: trace-server-migrator-1s3r
name: trace-server-migrator-1s3r-${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}
fail_ci_if_error: false
token: ${{ secrets.CODECOV_TOKEN }}
trace-server-migrator-2s2r:
name: Trace server migrator tests (2s2r)
needs:
- check-which-tests-to-run
if: needs.check-which-tests-to-run.outputs.trace_server_migrator_tests == 'true' || needs.check-which-tests-to-run.outputs.trace_server_distributed_tests == 'true' || github.ref == 'refs/heads/master'
timeout-minutes: 20
runs-on: ubuntu-latest
strategy:
matrix:
python-version-major: ["3"]
python-version-minor: ["12"]
fail-fast: false
steps:
- name: Checkout
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
- name: Start 2s2r ClickHouse cluster
working-directory: tests/trace_server/conftest_lib/topologies
run: |
docker compose -f 2s2r.yaml up -d --wait
# Extra belt-and-suspenders health probe. 2s2r entrypoint is on 8131.
for i in $(seq 1 60); do
wget -q -O- 'http://localhost:8131/ping' && break || sleep 2
done
docker compose -f 2s2r.yaml exec -T ch-s1-r1 \
clickhouse-client -q "SELECT host_name, shard_num, replica_num, errors_count FROM system.clusters WHERE cluster='weave_cluster' FORMAT PrettyCompact"
- name: Set up Python ${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
with:
python-version: ${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install nox uv
- name: Run migrator functional tests
env:
CI: 1
# 2s2r entrypoint is on host port 8131 (1s3r uses 8130).
# See 1s3r job for why only KEEPER_PORT is set.
WF_CLICKHOUSE_KEEPER_PORT: 8131
DD_TRACE_ENABLED: false
run: |
nox -e "tests-${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}(shard='trace_server_migrator')"
- name: Capture ClickHouse logs on failure
if: failure()
working-directory: tests/trace_server/conftest_lib/topologies
run: |
docker compose -f 2s2r.yaml logs --no-color --tail=1000 > ch-logs-2s2r.txt || true
- name: Upload ClickHouse logs artifact
if: failure()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: ch-logs-2s2r-${{ github.run_id }}-${{ github.run_attempt }}
path: tests/trace_server/conftest_lib/topologies/ch-logs-2s2r.txt
retention-days: 7
if-no-files-found: ignore
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5
with:
files: ./coverage.xml
flags: trace-server-migrator-2s2r
name: trace-server-migrator-2s2r-${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}
fail_ci_if_error: false
token: ${{ secrets.CODECOV_TOKEN }}
trace-tests-matrix-check: # This job does nothing and is only used for the branch protection
if: always()
needs:
- trace-tests
- trace_no_server
- trace-server-migrator-1s1r
- trace-server-migrator-1s3r
- trace-server-migrator-2s2r
runs-on: ubuntu-latest
steps:
- name: Passes if all trace-tests and trace_no_server jobs succeeded
uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # release/v1
with:
jobs: ${{ toJSON(needs) }}
# The 1s3r and 2s2r jobs are path-filtered and will be "skipped"
# on PRs that don't touch migrator or distributed surfaces.
# This action treats skipped as failing by default; whitelist
# them here so a skipped-because-gated job doesn't wedge merge.
allowed-skips: trace-server-migrator-1s3r,trace-server-migrator-2s2r
# ==== Windows Jobs ====
windows_trace_no_server:
name: Windows Trace non-trace server tests
timeout-minutes: 15
runs-on: windows-latest
strategy:
matrix:
python-version-major: ["3"]
python-version-minor: [
"10",
"11",
"13",
#
]
nox-shard: ["trace_no_server"]
fail-fast: false
steps:
- name: Checkout
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
- name: Set up Python ${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
with:
python-version: ${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}
- name: Set up uv (cached)
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
version: "0.11.19"
enable-cache: true
cache-local-path: ${{ runner.temp }}\uv-cache
cache-dependency-glob: "**/pyproject.toml"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install nox
- name: Run nox (Non Trace Server)
env:
DD_TRACE_ENABLED: "false"
run: |
nox -e "tests-${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}(shard='${{ matrix.nox-shard }}')" -- -m "not trace_server"
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5
with:
files: ./coverage.xml
flags: windows-trace-no-server
name: windows-trace-no-server-${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}
fail_ci_if_error: false
token: ${{ secrets.CODECOV_TOKEN }}
windows-trace-tests-matrix-check: # This job does nothing and is only used for the branch protection
if: always()
needs:
- windows_trace_no_server
runs-on: ubuntu-latest
steps:
- name: Passes if all windows_trace_no_server jobs succeeded
uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # release/v1
with:
jobs: ${{ toJSON(needs) }}