fix(canvas): apply black formatting, import ordering, and remove unused type: ignore

The source type enum should be added alongside its connector registration,
not as a premature scaffold that requires an expected_unmapped test hack.

backend/onyx/connectors/canvas/client.py

@@ -0,0 +1,192 @@
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any
+from urllib.parse import urlparse
+
+from onyx.connectors.cross_connector_utils.rate_limit_wrapper import (
+    rl_requests,
+)
+from onyx.error_handling.error_codes import OnyxErrorCode
+from onyx.error_handling.exceptions import OnyxError
+
+logger = logging.getLogger(__name__)
+
+# Requests timeout in seconds.
+_CANVAS_CALL_TIMEOUT: int = 30
+_CANVAS_API_VERSION: str = "/api/v1"
+# Matches the "next" URL in a Canvas Link header, e.g.:
+#   <https://canvas.example.com/api/v1/courses?page=2>; rel="next"
+# Captures the URL inside the angle brackets.
+_NEXT_LINK_PATTERN: re.Pattern[str] = re.compile(r'<([^>]+)>;\s*rel="next"')
+
+
+_STATUS_TO_ERROR_CODE: dict[int, OnyxErrorCode] = {
+    401: OnyxErrorCode.CREDENTIAL_EXPIRED,
+    403: OnyxErrorCode.INSUFFICIENT_PERMISSIONS,
+    404: OnyxErrorCode.BAD_GATEWAY,
+    429: OnyxErrorCode.RATE_LIMITED,
+}
+
+
+def _error_code_for_status(status_code: int) -> OnyxErrorCode:
+    """Map an HTTP status code to the appropriate OnyxErrorCode.
+
+    Expects a >= 400 status code. Known codes (401, 403, 404, 429) are
+    mapped to specific error codes; all other codes (unrecognised 4xx
+    and 5xx) map to BAD_GATEWAY as unexpected upstream errors.
+    """
+    if status_code in _STATUS_TO_ERROR_CODE:
+        return _STATUS_TO_ERROR_CODE[status_code]
+    return OnyxErrorCode.BAD_GATEWAY
+
+
+class CanvasApiClient:
+    def __init__(
+        self,
+        bearer_token: str,
+        canvas_base_url: str,
+    ) -> None:
+        parsed_base = urlparse(canvas_base_url)
+        if not parsed_base.hostname:
+            raise ValueError("canvas_base_url must include a valid host")
+        if parsed_base.scheme != "https":
+            raise ValueError("canvas_base_url must use https")
+
+        self._bearer_token = bearer_token
+        self.base_url = (
+            canvas_base_url.rstrip("/").removesuffix(_CANVAS_API_VERSION)
+            + _CANVAS_API_VERSION
+        )
+        # Hostname is already validated above; reuse parsed_base instead
+        # of re-parsing.  Used by _parse_next_link to validate pagination URLs.
+        self._expected_host: str = parsed_base.hostname
+
+    def get(
+        self,
+        endpoint: str = "",
+        params: dict[str, Any] | None = None,
+        full_url: str | None = None,
+    ) -> tuple[Any, str | None]:
+        """Make a GET request to the Canvas API.
+
+        Returns a tuple of (json_body, next_url).
+        next_url is parsed from the Link header and is None if there are no more pages.
+        If full_url is provided, it is used directly (for following pagination links).
+
+        Security note: full_url must only be set to values returned by
+        ``_parse_next_link``, which validates the host against the configured
+        Canvas base URL.  Passing an arbitrary URL would leak the bearer token.
+        """
+        # full_url is used when following pagination (Canvas returns the
+        # next-page URL in the Link header).  For the first request we build
+        # the URL from the endpoint name instead.
+        url = full_url if full_url else self._build_url(endpoint)
+        headers = self._build_headers()
+
+        response = rl_requests.get(
+            url,
+            headers=headers,
+            params=params if not full_url else None,
+            timeout=_CANVAS_CALL_TIMEOUT,
+        )
+
+        try:
+            response_json = response.json()
+        except ValueError as e:
+            if response.status_code < 300:
+                raise OnyxError(
+                    OnyxErrorCode.BAD_GATEWAY,
+                    detail=f"Invalid JSON in Canvas response: {e}",
+                )
+            logger.warning(
+                "Failed to parse JSON from Canvas error response (status=%d): %s",
+                response.status_code,
+                e,
+            )
+            response_json = {}
+
+        if response.status_code >= 400:
+            # Try to extract the most specific error message from the
+            # Canvas response body.  Canvas uses three different shapes
+            # depending on the endpoint and error type:
+            default_error: str = response.reason or f"HTTP {response.status_code}"
+            error = default_error
+            if isinstance(response_json, dict):
+                # Shape 1: {"error": {"message": "Not authorized"}}
+                error_field = response_json.get("error")
+                if isinstance(error_field, dict):
+                    response_error = error_field.get("message", "")
+                    if response_error:
+                        error = response_error
+                # Shape 2: {"error": "Invalid access token"}
+                elif isinstance(error_field, str):
+                    error = error_field
+                # Shape 3: {"errors": [{"message": "..."}]}
+                # Used for validation errors.  Only use as fallback if
+                # we didn't already find a more specific message above.
+                if error == default_error:
+                    errors_list = response_json.get("errors")
+                    if isinstance(errors_list, list) and errors_list:
+                        first_error = errors_list[0]
+                        if isinstance(first_error, dict):
+                            msg = first_error.get("message", "")
+                            if msg:
+                                error = msg
+            raise OnyxError(
+                _error_code_for_status(response.status_code),
+                detail=error,
+                status_code_override=response.status_code,
+            )
+
+        next_url = self._parse_next_link(response.headers.get("Link", ""))
+        return response_json, next_url
+
+    def _parse_next_link(self, link_header: str) -> str | None:
+        """Extract the 'next' URL from a Canvas Link header.
+
+        Only returns URLs whose host matches the configured Canvas base URL
+        to prevent leaking the bearer token to arbitrary hosts.
+        """
+        expected_host = self._expected_host
+        for match in _NEXT_LINK_PATTERN.finditer(link_header):
+            url = match.group(1)
+            parsed_url = urlparse(url)
+            if parsed_url.hostname != expected_host:
+                raise OnyxError(
+                    OnyxErrorCode.BAD_GATEWAY,
+                    detail=(
+                        "Canvas pagination returned an unexpected host "
+                        f"({parsed_url.hostname}); expected {expected_host}"
+                    ),
+                )
+            if parsed_url.scheme != "https":
+                raise OnyxError(
+                    OnyxErrorCode.BAD_GATEWAY,
+                    detail=(
+                        "Canvas pagination link must use https, "
+                        f"got {parsed_url.scheme!r}"
+                    ),
+                )
+            return url
+        return None
+
+    def _build_headers(self) -> dict[str, str]:
+        """Return the Authorization header with the bearer token."""
+        return {"Authorization": f"Bearer {self._bearer_token}"}
+
+    def _build_url(self, endpoint: str) -> str:
+        """Build a full Canvas API URL from an endpoint path.
+
+        Assumes endpoint is non-empty (e.g. ``"courses"``, ``"announcements"``).
+        Only called on a first request, endpoint must be set for first request.
+        Verify endpoint exists in case of future changes where endpoint might be optional.
+        Leading slashes are stripped to avoid double-slash in the result.
+        self.base_url is already normalized with no trailing slash.
+        """
+        final_url = self.base_url
+        clean_endpoint = endpoint.lstrip("/")
+        if clean_endpoint:
+            final_url += "/" + clean_endpoint
+        return final_url

backend/onyx/connectors/canvas/connector.py

@@ -0,0 +1,74 @@
+from typing import Literal
+from typing import TypeAlias
+
+from pydantic import BaseModel
+
+from onyx.connectors.models import ConnectorCheckpoint
+
+
+class CanvasCourse(BaseModel):
+    id: int
+    name: str
+    course_code: str
+    created_at: str
+    workflow_state: str
+
+
+class CanvasPage(BaseModel):
+    page_id: int
+    url: str
+    title: str
+    body: str | None = None
+    created_at: str
+    updated_at: str
+    course_id: int
+
+
+class CanvasAssignment(BaseModel):
+    id: int
+    name: str
+    description: str | None = None
+    html_url: str
+    course_id: int
+    created_at: str
+    updated_at: str
+    due_at: str | None = None
+
+
+class CanvasAnnouncement(BaseModel):
+    id: int
+    title: str
+    message: str | None = None
+    html_url: str
+    posted_at: str | None = None
+    course_id: int
+
+
+CanvasStage: TypeAlias = Literal["pages", "assignments", "announcements"]
+
+
+class CanvasConnectorCheckpoint(ConnectorCheckpoint):
+    """Checkpoint state for resumable Canvas indexing.
+
+    Fields:
+        course_ids: Materialized list of course IDs to process.
+        current_course_index: Index into course_ids for current course.
+        stage: Which item type we're processing for the current course.
+        next_url: Pagination cursor within the current stage. None means
+            start from the first page; a URL means resume from that page.
+
+    Invariant:
+        If current_course_index is incremented, stage must be reset to
+        "pages" and next_url must be reset to None.
+    """
+
+    course_ids: list[int] = []
+    current_course_index: int = 0
+    stage: CanvasStage = "pages"
+    next_url: str | None = None
+
+    def advance_course(self) -> None:
+        """Move to the next course and reset within-course state."""
+        self.current_course_index += 1
+        self.stage = "pages"
+        self.next_url = None

backend/tests/unit/onyx/connectors/canvas/test_canvas_connector.py

@@ -0,0 +1,381 @@
+"""Tests for Canvas connector — client (PR1)."""
+
+from typing import Any
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+import pytest
+
+from onyx.connectors.canvas.client import CanvasApiClient
+from onyx.error_handling.exceptions import OnyxError
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+FAKE_BASE_URL = "https://myschool.instructure.com"
+FAKE_TOKEN = "fake-canvas-token"
+
+
+def _mock_response(
+    status_code: int = 200,
+    json_data: Any = None,
+    link_header: str = "",
+) -> MagicMock:
+    """Create a mock HTTP response with status, json, and Link header."""
+    resp = MagicMock()
+    resp.status_code = status_code
+    resp.reason = "OK" if status_code < 300 else "Error"
+    resp.json.return_value = json_data if json_data is not None else []
+    resp.headers = {"Link": link_header}
+    return resp
+
+
+# ---------------------------------------------------------------------------
+# CanvasApiClient.__init__ tests
+# ---------------------------------------------------------------------------
+
+
+class TestCanvasApiClientInit:
+    def test_success(self) -> None:
+        client = CanvasApiClient(
+            bearer_token=FAKE_TOKEN,
+            canvas_base_url=FAKE_BASE_URL,
+        )
+
+        expected_base_url = f"{FAKE_BASE_URL}/api/v1"
+        expected_host = "myschool.instructure.com"
+
+        assert client.base_url == expected_base_url
+        assert client._expected_host == expected_host
+
+    def test_normalizes_trailing_slash(self) -> None:
+        client = CanvasApiClient(
+            bearer_token=FAKE_TOKEN,
+            canvas_base_url=f"{FAKE_BASE_URL}/",
+        )
+
+        expected_base_url = f"{FAKE_BASE_URL}/api/v1"
+
+        assert client.base_url == expected_base_url
+
+    def test_normalizes_existing_api_v1(self) -> None:
+        client = CanvasApiClient(
+            bearer_token=FAKE_TOKEN,
+            canvas_base_url=f"{FAKE_BASE_URL}/api/v1",
+        )
+
+        expected_base_url = f"{FAKE_BASE_URL}/api/v1"
+
+        assert client.base_url == expected_base_url
+
+    def test_rejects_non_https_scheme(self) -> None:
+        with pytest.raises(ValueError, match="must use https"):
+            CanvasApiClient(
+                bearer_token=FAKE_TOKEN,
+                canvas_base_url="ftp://myschool.instructure.com",
+            )
+
+    def test_rejects_http(self) -> None:
+        with pytest.raises(ValueError, match="must use https"):
+            CanvasApiClient(
+                bearer_token=FAKE_TOKEN,
+                canvas_base_url="http://myschool.instructure.com",
+            )
+
+    def test_rejects_missing_host(self) -> None:
+        with pytest.raises(ValueError, match="must include a valid host"):
+            CanvasApiClient(
+                bearer_token=FAKE_TOKEN,
+                canvas_base_url="https://",
+            )
+
+
+# ---------------------------------------------------------------------------
+# CanvasApiClient._build_url tests
+# ---------------------------------------------------------------------------
+
+
+class TestBuildUrl:
+    def setup_method(self) -> None:
+        self.client = CanvasApiClient(
+            bearer_token=FAKE_TOKEN,
+            canvas_base_url=FAKE_BASE_URL,
+        )
+
+    def test_appends_endpoint(self) -> None:
+        result = self.client._build_url("courses")
+        expected = f"{FAKE_BASE_URL}/api/v1/courses"
+
+        assert result == expected
+
+    def test_strips_leading_slash_from_endpoint(self) -> None:
+        result = self.client._build_url("/courses")
+        expected = f"{FAKE_BASE_URL}/api/v1/courses"
+
+        assert result == expected
+
+
+# ---------------------------------------------------------------------------
+# CanvasApiClient._build_headers tests
+# ---------------------------------------------------------------------------
+
+
+class TestBuildHeaders:
+    def setup_method(self) -> None:
+        self.client = CanvasApiClient(
+            bearer_token=FAKE_TOKEN,
+            canvas_base_url=FAKE_BASE_URL,
+        )
+
+    def test_returns_bearer_auth(self) -> None:
+        result = self.client._build_headers()
+        expected = {"Authorization": f"Bearer {FAKE_TOKEN}"}
+
+        assert result == expected
+
+
+# ---------------------------------------------------------------------------
+# CanvasApiClient.get tests
+# ---------------------------------------------------------------------------
+
+
+class TestGet:
+    def setup_method(self) -> None:
+        self.client = CanvasApiClient(
+            bearer_token=FAKE_TOKEN,
+            canvas_base_url=FAKE_BASE_URL,
+        )
+
+    @patch("onyx.connectors.canvas.client.rl_requests")
+    def test_success_returns_json_and_next_url(self, mock_requests: MagicMock) -> None:
+        next_link = f"<{FAKE_BASE_URL}/api/v1/courses?page=2>; " 'rel="next"'
+        mock_requests.get.return_value = _mock_response(
+            json_data=[{"id": 1}], link_header=next_link
+        )
+
+        data, next_url = self.client.get("courses")
+
+        expected_data = [{"id": 1}]
+        expected_next = f"{FAKE_BASE_URL}/api/v1/courses?page=2"
+
+        assert data == expected_data
+        assert next_url == expected_next
+
+    @patch("onyx.connectors.canvas.client.rl_requests")
+    def test_success_no_next_page(self, mock_requests: MagicMock) -> None:
+        mock_requests.get.return_value = _mock_response(json_data=[{"id": 1}])
+
+        data, next_url = self.client.get("courses")
+
+        assert data == [{"id": 1}]
+        assert next_url is None
+
+    @patch("onyx.connectors.canvas.client.rl_requests")
+    def test_raises_on_error_status(self, mock_requests: MagicMock) -> None:
+        mock_requests.get.return_value = _mock_response(403, {})
+
+        with pytest.raises(OnyxError) as exc_info:
+            self.client.get("courses")
+
+        assert exc_info.value.status_code == 403
+
+    @patch("onyx.connectors.canvas.client.rl_requests")
+    def test_raises_on_404(self, mock_requests: MagicMock) -> None:
+        mock_requests.get.return_value = _mock_response(404, {})
+
+        with pytest.raises(OnyxError) as exc_info:
+            self.client.get("courses")
+
+        assert exc_info.value.status_code == 404
+
+    @patch("onyx.connectors.canvas.client.rl_requests")
+    def test_raises_on_429(self, mock_requests: MagicMock) -> None:
+        mock_requests.get.return_value = _mock_response(429, {})
+
+        with pytest.raises(OnyxError) as exc_info:
+            self.client.get("courses")
+
+        assert exc_info.value.status_code == 429
+
+    @patch("onyx.connectors.canvas.client.rl_requests")
+    def test_skips_params_when_using_full_url(self, mock_requests: MagicMock) -> None:
+        mock_requests.get.return_value = _mock_response(json_data=[])
+        full = f"{FAKE_BASE_URL}/api/v1/courses?page=2"
+
+        self.client.get(params={"per_page": "100"}, full_url=full)
+
+        _, kwargs = mock_requests.get.call_args
+        assert kwargs["params"] is None
+
+    @patch("onyx.connectors.canvas.client.rl_requests")
+    def test_error_extracts_message_from_error_dict(
+        self, mock_requests: MagicMock
+    ) -> None:
+        """Shape 1: {"error": {"message": "Not authorized"}}"""
+        mock_requests.get.return_value = _mock_response(
+            403, {"error": {"message": "Not authorized"}}
+        )
+
+        with pytest.raises(OnyxError) as exc_info:
+            self.client.get("courses")
+
+        result = exc_info.value.detail
+        expected = "Not authorized"
+
+        assert result == expected
+
+    @patch("onyx.connectors.canvas.client.rl_requests")
+    def test_error_extracts_message_from_error_string(
+        self, mock_requests: MagicMock
+    ) -> None:
+        """Shape 2: {"error": "Invalid access token"}"""
+        mock_requests.get.return_value = _mock_response(
+            401, {"error": "Invalid access token"}
+        )
+
+        with pytest.raises(OnyxError) as exc_info:
+            self.client.get("courses")
+
+        result = exc_info.value.detail
+        expected = "Invalid access token"
+
+        assert result == expected
+
+    @patch("onyx.connectors.canvas.client.rl_requests")
+    def test_error_extracts_message_from_errors_list(
+        self, mock_requests: MagicMock
+    ) -> None:
+        """Shape 3: {"errors": [{"message": "Invalid query"}]}"""
+        mock_requests.get.return_value = _mock_response(
+            400, {"errors": [{"message": "Invalid query"}]}
+        )
+
+        with pytest.raises(OnyxError) as exc_info:
+            self.client.get("courses")
+
+        result = exc_info.value.detail
+        expected = "Invalid query"
+
+        assert result == expected
+
+    @patch("onyx.connectors.canvas.client.rl_requests")
+    def test_error_dict_takes_priority_over_errors_list(
+        self, mock_requests: MagicMock
+    ) -> None:
+        """When both error shapes are present, error dict wins."""
+        mock_requests.get.return_value = _mock_response(
+            403, {"error": "Specific error", "errors": [{"message": "Generic"}]}
+        )
+
+        with pytest.raises(OnyxError) as exc_info:
+            self.client.get("courses")
+
+        result = exc_info.value.detail
+        expected = "Specific error"
+
+        assert result == expected
+
+    @patch("onyx.connectors.canvas.client.rl_requests")
+    def test_error_falls_back_to_reason_when_no_json_message(
+        self, mock_requests: MagicMock
+    ) -> None:
+        """Empty error body falls back to response.reason."""
+        mock_requests.get.return_value = _mock_response(500, {})
+
+        with pytest.raises(OnyxError) as exc_info:
+            self.client.get("courses")
+
+        result = exc_info.value.detail
+        expected = "Error"  # from _mock_response's reason for >= 300
+
+        assert result == expected
+
+    @patch("onyx.connectors.canvas.client.rl_requests")
+    def test_invalid_json_on_success_raises(self, mock_requests: MagicMock) -> None:
+        """Invalid JSON on a 2xx response raises OnyxError."""
+        resp = MagicMock()
+        resp.status_code = 200
+        resp.json.side_effect = ValueError("No JSON")
+        resp.headers = {"Link": ""}
+        mock_requests.get.return_value = resp
+
+        with pytest.raises(OnyxError, match="Invalid JSON"):
+            self.client.get("courses")
+
+    @patch("onyx.connectors.canvas.client.rl_requests")
+    def test_invalid_json_on_error_falls_back_to_reason(
+        self, mock_requests: MagicMock
+    ) -> None:
+        """Invalid JSON on a 4xx response falls back to response.reason."""
+        resp = MagicMock()
+        resp.status_code = 500
+        resp.reason = "Internal Server Error"
+        resp.json.side_effect = ValueError("No JSON")
+        resp.headers = {"Link": ""}
+        mock_requests.get.return_value = resp
+
+        with pytest.raises(OnyxError) as exc_info:
+            self.client.get("courses")
+
+        result = exc_info.value.detail
+        expected = "Internal Server Error"
+
+        assert result == expected
+
+
+# ---------------------------------------------------------------------------
+# CanvasApiClient._parse_next_link tests
+# ---------------------------------------------------------------------------
+
+
+class TestParseNextLink:
+    def setup_method(self) -> None:
+        self.client = CanvasApiClient(
+            bearer_token=FAKE_TOKEN,
+            canvas_base_url="https://canvas.example.com",
+        )
+
+    def test_found(self) -> None:
+        header = '<https://canvas.example.com/api/v1/courses?page=2>; rel="next"'
+
+        result = self.client._parse_next_link(header)
+        expected = "https://canvas.example.com/api/v1/courses?page=2"
+
+        assert result == expected
+
+    def test_not_found(self) -> None:
+        header = '<https://canvas.example.com/api/v1/courses?page=1>; rel="current"'
+
+        result = self.client._parse_next_link(header)
+
+        assert result is None
+
+    def test_empty(self) -> None:
+        result = self.client._parse_next_link("")
+
+        assert result is None
+
+    def test_multiple_rels(self) -> None:
+        header = (
+            '<https://canvas.example.com/api/v1/courses?page=1>; rel="current", '
+            '<https://canvas.example.com/api/v1/courses?page=2>; rel="next"'
+        )
+
+        result = self.client._parse_next_link(header)
+        expected = "https://canvas.example.com/api/v1/courses?page=2"
+
+        assert result == expected
+
+    def test_rejects_host_mismatch(self) -> None:
+        header = '<https://evil.example.com/api/v1/courses?page=2>; rel="next"'
+
+        with pytest.raises(OnyxError, match="unexpected host"):
+            self.client._parse_next_link(header)
+
+    def test_rejects_non_https_link(self) -> None:
+        header = '<http://canvas.example.com/api/v1/courses?page=2>; rel="next"'
+
+        with pytest.raises(OnyxError, match="must use https"):
+            self.client._parse_next_link(header)