ci.yml

# PR gate — proves every change still builds, passes the full dbt data-test and
# unit-test suite, and lints clean. Counts are deliberately not written down here:
# `dbt build` runs whatever the project defines, so a number in a comment can only
# go stale. The current totals are in the README's Testing & CI table.
#
# Triggers:
#   - pull_request to main: every PR is gated
#   - workflow_dispatch: manual trigger from any branch (debugging)
#
# What it does:
#   1. Set up Python 3.13 + the pinned dbt stack
#   2. Install dbt packages (dbt_utils, dbt_expectations, dbt_date)
#   3. Stage profiles.yml from the committed example
#   4. Python unit tests — downloader, dependency imports, FastAPI API suite,
#      and the Dagster definitions smoke test (needs dbt deps + profiles to
#      parse the manifest, hence this ordering)
#   5. Download two years of Land Registry data (--sample, enough for YoY)
#   6. Load it into DuckDB
#   7. dbt source freshness — proves the loaded source is current enough
#   8. dbt seed + dbt build — all marts + every data test and unit test on sample
#   9. Web golden tests, lint, and production build against the local API
#  10. sqlfluff lint — model SQL must satisfy .sqlfluff rules
#
# Branch protection on main requires the "build-and-test" check to pass before
# merging. See repo Settings → Branches.
#
# Why a sample, not the full multi-year Parquet:
#   - The full pull is an order of magnitude larger than the sample and would add
#     minutes of download and load time to every PR. Sizes and years are not
#     written down here because both move with the source; scripts/download_raw.py
#     is the authority. The sample exercises the same SQL paths, runs the whole
#     data-test suite, and proves YoY logic is non-vacuous. Row-count test bounds
#     were calibrated to pass on either full or sample data.

name: CI

on:
  pull_request:
    branches: [main]
  # Also run on push to main so the README CI badge reflects the live state
  # of trunk. Branch protection ensures every commit on main first passed
  # the same workflow on its PR; this push run is for visibility, not gating.
  push:
    branches: [main]
  workflow_dispatch:

# Cancel superseded runs on the same PR — saves CI minutes when pushing
# fixup commits in quick succession.
concurrency:
  group: ci-${{ github.ref }}
  cancel-in-progress: true

jobs:

  build-and-test:
    # No job-level `name:` here on purpose — GitHub reports the check status
    # under the job ID (`build-and-test`) when no display name is set, which
    # is what the branch-protection ruleset on `main` waits for.
    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Python 3.13
        uses: actions/setup-python@v5
        with:
          python-version: "3.13"
          cache: pip

      - name: Set up Node.js 22
        uses: actions/setup-node@v4
        with:
          node-version: "22"
          cache: npm
          cache-dependency-path: web/package-lock.json

      - name: Install Python dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt

      - name: Install dbt packages
        run: dbt deps

      - name: Stage profiles.yml from committed example
        run: |
          mkdir -p ~/.dbt
          cp profiles.yml.example ~/.dbt/profiles.yml

      - name: Python unit tests (downloader, imports, API suite, Dagster defs)
        # Runs after `dbt deps` + profiles staging: the orchestration smoke
        # test parses the dbt manifest to load the Dagster definitions.
        run: python -m unittest discover -s tests -p "test_*.py"

      - name: API container starts with its runtime contract
        run: |
          docker build --file api/Dockerfile --tag housing-api-ci .
          docker run --detach --name housing-api-ci --publish 127.0.0.1:8080:8080 housing-api-ci
          trap 'docker logs housing-api-ci; docker rm --force housing-api-ci' EXIT
          for attempt in {1..20}; do
            if curl --fail --silent http://127.0.0.1:8080/healthz; then
              exit 0
            fi
            sleep 1
          done
          exit 1

      - name: dbt scoring contract rejects invalid weights
        run: |
          if dbt compile --target ci --select rpt_neighbourhood_score --vars '{score_weight_affordability: 6}'; then
            echo "Expected the scoring contract to reject weight 6."
            exit 1
          fi

      - name: Download Land Registry sample (2 years)
        run: python scripts/download_raw.py --sample

      - name: Load sample into DuckDB
        run: python scripts/load_to_duckdb.py

      - name: dbt source freshness
        run: dbt source freshness --target dev

      - name: dbt seed
        run: dbt seed --target dev --threads 1

      - name: dbt build (all marts, data tests, and unit tests)
        run: dbt build --target dev --threads 1

      - name: Install web dependencies
        run: npm ci --prefix web

      - name: Web scoring golden tests and lint
        run: |
          npm test --prefix web
          npm run lint --prefix web

      - name: Web production build against local API
        run: |
          python -m uvicorn api.main:app --host 127.0.0.1 --port 8000 &
          api_pid=$!
          trap 'kill "$api_pid"' EXIT
          for attempt in {1..20}; do
            curl --fail --silent http://127.0.0.1:8000/healthz && break
            sleep 1
          done
          API_BASE_URL=http://127.0.0.1:8000 npm run build --prefix web

      - name: sqlfluff lint (model SQL only)
        # Hard gate: a style violation in models/ fails the PR check, same as
        # a failing dbt test. The lint config (.sqlfluff) excludes a handful
        # of noisy rules — see the comments in that file. analyses/ and
        # macros/_generated/ are excluded via .sqlfluffignore.
        #
        # The dbt templater needs profiles.yml at the location specified by
        # .sqlfluff (`profiles_dir = ./`). Copy it into the project root just
        # for the lint step — gitignored, never committed.
        run: |
          cp profiles.yml.example profiles.yml
          sqlfluff lint models/ --dialect duckdb --templater dbt
          rm profiles.yml