diff --git a/src/basic_memory/alembic/versions/u4t5e6m7p8o9_add_memory_time_index_table.py b/src/basic_memory/alembic/versions/u4t5e6m7p8o9_add_memory_time_index_table.py new file mode 100644 index 000000000..b445520d1 --- /dev/null +++ b/src/basic_memory/alembic/versions/u4t5e6m7p8o9_add_memory_time_index_table.py @@ -0,0 +1,103 @@ +"""Add memory_time_index table + +Revision ID: u4t5e6m7p8o9 +Revises: t3n4o5t6e7s8 +Create Date: 2026-08-31 12:00:00.000000 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + + +# revision identifiers, used by Alembic. +revision: str = "u4t5e6m7p8o9" +down_revision: Union[str, None] = "t3n4o5t6e7s8" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Create memory_time_index: the projection of authored valid time (SPEC-82). + + Rows derive from temporal qualifiers written in canonical markdown + (``@effective[2026-06-10,2026-07-27)``). Like observations and sections they are + rebuilt under the note_content generation fence on every (re)index and removed with + the entity, so the table is always reproducible from the notes. + + Every column is a portable scalar, and every type used here renders on both SQLite + and PostgreSQL, so this migration needs no dialect branching. Bounds are canonical + fixed-width text rather than DATE/TIMESTAMP: a date bound must never acquire a time + of day or a timezone (SQLAlchemy's SQLite DateTime silently drops an offset and + stores the wrong instant), and fixed-width canonical text makes byte-lexicographic + order chronological, so one identical predicate serves both backends. Native + PostgreSQL range columns remain a later addition generated from these columns. + + ``source_id`` carries no foreign key by design: it addresses whichever table + ``source_type`` names (``observation`` today). Referential lifecycle rides on + ``entity_id``'s cascade plus the fenced replace instead. + + Only ``ix_memory_time_index_lookup`` indexes the filter columns. The bound values + are deliberately unindexed, and this table is *not* always driven by a full-text + candidate set: a valid-time filter counts as criteria on its own + (``SearchQuery.no_criteria``), so a temporal-only search scans the bound columns + for every row matching project + kind + axis. That is an acceptable scan at + expected sizes -- one row per authored qualifier, so thousands, not millions. If + temporal-only queries ever become a hot path, the answer is a native PostgreSQL + range column with a GiST index, not a btree over these text bounds. + """ + op.create_table( + "memory_time_index", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("project_id", sa.Integer(), nullable=False), + sa.Column("entity_id", sa.Integer(), nullable=False), + sa.Column("source_type", sa.String(length=32), nullable=False), + sa.Column("source_id", sa.Integer(), nullable=False), + sa.Column("time_kind", sa.String(length=32), nullable=False), + sa.Column("range_axis", sa.String(length=16), nullable=False), + sa.Column("lower_value", sa.String(length=32), nullable=True), + sa.Column("upper_value", sa.String(length=32), nullable=True), + sa.Column("lower_inclusive", sa.Boolean(), nullable=False), + sa.Column("upper_inclusive", sa.Boolean(), nullable=False), + sa.Column("is_empty", sa.Boolean(), nullable=False, server_default=sa.false()), + sa.Column("extractor", sa.String(length=32), nullable=False), + sa.Column("source_text", sa.Text(), nullable=False), + sa.Column("assertion_metadata", sa.JSON(), nullable=True), + sa.ForeignKeyConstraint(["project_id"], ["project.id"]), + sa.ForeignKeyConstraint(["entity_id"], ["entity.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.CheckConstraint( + "range_axis IN ('date', 'instant')", + name="ck_memory_time_index_range_axis", + ), + sa.CheckConstraint( + "NOT is_empty OR (lower_value IS NULL AND upper_value IS NULL)", + name="ck_memory_time_index_empty_has_no_bounds", + ), + sa.CheckConstraint( + "(lower_value IS NOT NULL OR NOT lower_inclusive) " + "AND (upper_value IS NOT NULL OR NOT upper_inclusive)", + name="ck_memory_time_index_unbounded_is_exclusive", + ), + ) + op.create_index( + "ix_memory_time_index_lookup", + "memory_time_index", + ["project_id", "time_kind", "range_axis", "source_type", "source_id"], + unique=False, + ) + op.create_index( + "ix_memory_time_index_entity_id", + "memory_time_index", + ["entity_id"], + unique=False, + ) + + +def downgrade() -> None: + """Drop memory_time_index and its supporting indexes.""" + op.drop_index("ix_memory_time_index_entity_id", table_name="memory_time_index") + op.drop_index("ix_memory_time_index_lookup", table_name="memory_time_index") + op.drop_table("memory_time_index") diff --git a/src/basic_memory/alembic/versions/v5o6b7s8d9e0_add_observation_duplicate_index.py b/src/basic_memory/alembic/versions/v5o6b7s8d9e0_add_observation_duplicate_index.py new file mode 100644 index 000000000..d0ce6c648 --- /dev/null +++ b/src/basic_memory/alembic/versions/v5o6b7s8d9e0_add_observation_duplicate_index.py @@ -0,0 +1,49 @@ +"""Add the duplicate ordinal that keeps same-identity observations addressable. + +An observation's synthetic permalink is built from its category and content, both of +which survive the peel that strips a temporal qualifier and a (context) off the authored +line. Two observations differing only in one of those therefore shared one address, and +the permalink-keyed search index kept only the first -- so the second note's authored +valid time addressed an observation with no search row, and queries for its interval +found nothing (SPEC-82). + +This ordinal separates such twins. It defaults to 0, which is the value every existing +row takes and the value the first observation of any identity keeps, so no permalink that +resolves today changes. Search rows are derived state and are rebuilt from markdown, so +the second twin becomes addressable on the next index pass for that note. + +Revision ID: v5o6b7s8d9e0 +Revises: u4t5e6m7p8o9 +Create Date: 2026-09-02 10:00:00.000000 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "v5o6b7s8d9e0" +down_revision: Union[str, None] = "u4t5e6m7p8o9" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Add observation.duplicate_index, defaulting every existing row to 0.""" + with op.batch_alter_table("observation", schema=None) as batch_op: + batch_op.add_column( + sa.Column( + "duplicate_index", + sa.Integer(), + server_default=sa.text("0"), + nullable=False, + ) + ) + + +def downgrade() -> None: + """Remove the duplicate ordinal.""" + with op.batch_alter_table("observation", schema=None) as batch_op: + batch_op.drop_column("duplicate_index") diff --git a/src/basic_memory/api/v2/routers/search_router.py b/src/basic_memory/api/v2/routers/search_router.py index eaf618b89..49a015868 100644 --- a/src/basic_memory/api/v2/routers/search_router.py +++ b/src/basic_memory/api/v2/routers/search_router.py @@ -12,13 +12,16 @@ from fastapi import APIRouter, Depends, HTTPException, Path, Response import logfire -from basic_memory.api.v2.utils import to_search_results +from basic_memory import db +from basic_memory.api.v2.utils import load_temporal_metadata, to_search_results from basic_memory.deps import ( EntityServiceV2ExternalDep, + MemoryTimeIndexRepositoryV2ExternalDep, ProjectExternalIdPathDep, ReadCacheDep, SearchReindexSchedulerDep, SearchServiceV2ExternalDep, + SessionMakerDep, create_model_read_cache, ) from basic_memory.read_cache import ( @@ -83,6 +86,8 @@ async def search( query: SearchQuery, search_service: SearchServiceV2ExternalDep, entity_service: EntityServiceV2ExternalDep, + temporal_repository: MemoryTimeIndexRepositoryV2ExternalDep, + session_maker: SessionMakerDep, read_cache: SearchReadCacheDep, response: Response, project_id: str = Path(..., description="Project external UUID"), @@ -98,6 +103,8 @@ async def search( query: Search query parameters (text, filters, etc.) search_service: Search service scoped to project entity_service: Entity service scoped to project + temporal_repository: Valid-time projection, read to explain temporal matches + session_maker: Session factory for the temporal hydration read page: Page number for pagination page_size: Number of results per page @@ -105,6 +112,10 @@ async def search( SearchResponse with paginated search results """ response.headers["Accept-Query"] = "application/json" + # Read from the request rather than from the parsed filter: this is a plain field + # check that cannot raise, and reaching the hydration step below already proves + # the service accepted and executed the filter. + temporal_requested = query.has_temporal_filter() with logfire.span( "api.request.search", entrypoint="api", @@ -125,7 +136,9 @@ async def search( or query.categories or query.metadata_filters or query.file_path_prefix + or temporal_requested ), + has_temporal_filter=temporal_requested, ): cache_key = ReadCacheKey( project_id=project_id, @@ -202,7 +215,20 @@ async def search( phase="hydrate_results", result_count=len(results), ): - search_results = await to_search_results(entity_service, results) + # Trigger: the caller asked a valid-time question. + # Why: the assertions explain *why* each hit matched, but loading them + # costs a query, and a search with no temporal filter has nothing to + # explain -- so ordinary searches stay exactly as expensive as before. + # Outcome: temporal metadata rides along only on temporal searches. + temporal_by_source = {} + if temporal_requested: + async with db.scoped_session(session_maker) as session: + temporal_by_source = await load_temporal_metadata( + temporal_repository, session, results + ) + search_results = await to_search_results( + entity_service, results, temporal_by_source=temporal_by_source + ) with logfire.span( "api.search.search.build_response", domain="search", @@ -217,6 +243,9 @@ async def search( total=total, total_is_exact=exact_count_available, has_more=has_more, + # None, not False, when nothing was asked: an ordinary search + # payload stays exactly what it was before valid time existed. + temporal_applied=True if temporal_requested else None, ) cached.value = result return result diff --git a/src/basic_memory/api/v2/utils.py b/src/basic_memory/api/v2/utils.py index 853f2c76f..7ed3bfde1 100644 --- a/src/basic_memory/api/v2/utils.py +++ b/src/basic_memory/api/v2/utils.py @@ -1,7 +1,10 @@ +from collections import defaultdict +from collections.abc import Mapping from typing import Any, Protocol, Optional, List, Sequence import logfire from sqlalchemy.ext.asyncio import AsyncSession +from basic_memory.models import MemoryTimeIndex from basic_memory.repository.search_repository import SearchIndexRow from basic_memory.schemas.memory import ( EntitySummary, @@ -11,11 +14,17 @@ GraphContext, ContextResult, ) -from basic_memory.schemas.search import SearchItemType, SearchResult +from basic_memory.schemas.search import ( + SearchItemType, + SearchResult, + TemporalRangeValue, + TemporalResultMetadata, +) from basic_memory.services.context_service import ( ContextResultRow, ContextResult as ServiceContextResult, ) +from basic_memory.temporal import TemporalRange, TemporalRangeAxis class EntityBatchLookup(Protocol): @@ -32,6 +41,19 @@ class EntityServiceBatchLookup(Protocol): async def get_entities_by_id(self, ids: List[int]) -> Sequence[Any]: ... +class TemporalAssertionLookup(Protocol): + async def find_for_sources( + self, + session: AsyncSession, + sources: Sequence[tuple[str, int]], + ) -> Sequence[MemoryTimeIndex]: ... + + +# One page of hits, keyed by the (search row type, search row id) pair the projection +# addresses. Empty means "no valid-time metadata was loaded", never "none exists". +type TemporalMetadataBySource = Mapping[tuple[str, int], list[TemporalResultMetadata]] + + async def get_entities_by_id_lookup( entity_service: EntityServiceBatchLookup, entity_ids: Sequence[int], @@ -217,8 +239,71 @@ def to_summary( ) +def _temporal_result_metadata(row: MemoryTimeIndex) -> TemporalResultMetadata: + """Shape one projected assertion into the value a caller sees. + + Rebuilding the domain range from the stored scalars re-runs its invariants, so a + row that somehow violated them surfaces here instead of being rendered as a + plausible-looking interval. + """ + valid_during = TemporalRange( + axis=TemporalRangeAxis(row.range_axis), + lower=row.lower_value, + upper=row.upper_value, + lower_inclusive=row.lower_inclusive, + upper_inclusive=row.upper_inclusive, + is_empty=row.is_empty, + ) + return TemporalResultMetadata( + kind=row.time_kind, + valid_during=TemporalRangeValue( + axis=valid_during.axis.value, + literal=str(valid_during), + lower=valid_during.lower, + upper=valid_during.upper, + lower_inclusive=valid_during.lower_inclusive, + upper_inclusive=valid_during.upper_inclusive, + is_empty=valid_during.is_empty, + ), + source_text=row.source_text, + ) + + +async def load_temporal_metadata( + temporal_repository: TemporalAssertionLookup, + session: AsyncSession, + results: Sequence[SearchIndexRow], +) -> dict[tuple[str, int], list[TemporalResultMetadata]]: + """Load the authored valid-time assertions behind one page of search hits. + + Keyed on the search row's own ``(type, id)`` pair, which is exactly the address + the projection stores -- so an observation hit resolves to the assertions written + on that observation, not to its note's other assertions. + """ + sources = [(result.type, result.id) for result in results] + if not sources: + return {} + + with logfire.span( + "search.hydrate_results.fetch_temporal", + domain="search", + action="search", + phase="fetch_temporal", + result_count=len(sources), + ): + rows = await temporal_repository.find_for_sources(session, sources) + + by_source: defaultdict[tuple[str, int], list[TemporalResultMetadata]] = defaultdict(list) + for row in rows: + by_source[(row.source_type, row.source_id)].append(_temporal_result_metadata(row)) + return dict(by_source) + + async def to_search_results( - entity_service: EntityServiceBatchLookup, results: List[SearchIndexRow] + entity_service: EntityServiceBatchLookup, + results: List[SearchIndexRow], + *, + temporal_by_source: TemporalMetadataBySource | None = None, ) -> list[SearchResult]: with logfire.span( "search.hydrate_results", @@ -299,6 +384,11 @@ async def to_search_results( from_entity=from_entity.permalink if from_entity else None, to_entity=to_entity.permalink if to_entity else None, relation_type=result.relation_type, + temporal=( + temporal_by_source.get((result.type, result.id)) + if temporal_by_source + else None + ), ) ) return search_results diff --git a/src/basic_memory/config_models.py b/src/basic_memory/config_models.py index bcb9227ab..fb8b4991b 100644 --- a/src/basic_memory/config_models.py +++ b/src/basic_memory/config_models.py @@ -586,6 +586,15 @@ def __init__(self, **data: Any) -> None: ... gt=0, ) + # Spelled as a bare Literal rather than the `DateOrder` alias so `bm config set` + # keeps discovering it: CONFIGURABLE_FIELDS reads model_fields annotations, and a + # PEP 695 alias arrives there unresolved. `temporal.DateOrder` is the same union, + # and a test pins the two together. + date_order: Literal["YMD", "DMY", "MDY"] = Field( + default="YMD", + description="Component order used to read an ambiguous slash-formatted date in an authored temporal qualifier (e.g. '@10/07/2026'). YMD and DMY read that as 10 July 2026, MDY as 7 October 2026. ISO dates like '2026-07-10' are never re-guessed.", + ) + kebab_filenames: bool = Field( default=False, description="Format for generated filenames. False preserves spaces and special chars, True converts them to hyphens for consistency with permalinks", diff --git a/src/basic_memory/deps/__init__.py b/src/basic_memory/deps/__init__.py index d45c6e18f..f274c88b8 100644 --- a/src/basic_memory/deps/__init__.py +++ b/src/basic_memory/deps/__init__.py @@ -51,6 +51,8 @@ ObservationRepositoryV2ExternalDep, get_relation_repository_v2_external, RelationRepositoryV2ExternalDep, + get_memory_time_index_repository_v2_external, + MemoryTimeIndexRepositoryV2ExternalDep, get_search_repository_v2_external, SearchRepositoryV2ExternalDep, ) @@ -140,6 +142,8 @@ "ObservationRepositoryV2ExternalDep", "get_relation_repository_v2_external", "RelationRepositoryV2ExternalDep", + "get_memory_time_index_repository_v2_external", + "MemoryTimeIndexRepositoryV2ExternalDep", "get_search_repository_v2_external", "SearchRepositoryV2ExternalDep", # Services diff --git a/src/basic_memory/deps/repositories.py b/src/basic_memory/deps/repositories.py index ccc325880..7dd583318 100644 --- a/src/basic_memory/deps/repositories.py +++ b/src/basic_memory/deps/repositories.py @@ -18,6 +18,7 @@ from basic_memory.deps.db import SessionMakerDep from basic_memory.deps.projects import ProjectExternalIdPathDep from basic_memory.repository.entity_repository import EntityRepository +from basic_memory.repository.memory_time_index_repository import MemoryTimeIndexRepository from basic_memory.repository.observation_repository import ObservationRepository from basic_memory.repository.relation_repository import RelationRepository from basic_memory.repository.search_repository import SearchRepository, create_search_repository @@ -68,6 +69,21 @@ async def get_relation_repository_v2_external( ] +# --- Temporal Projection Repository --- + + +async def get_memory_time_index_repository_v2_external( + project_id: ProjectExternalIdPathDep, +) -> MemoryTimeIndexRepository: + """Create a MemoryTimeIndexRepository instance for v2 API (uses external_id).""" + return MemoryTimeIndexRepository(project_id=project_id) + + +MemoryTimeIndexRepositoryV2ExternalDep = Annotated[ + MemoryTimeIndexRepository, Depends(get_memory_time_index_repository_v2_external) +] + + # --- Search Repository --- diff --git a/src/basic_memory/indexing/accepted_note_write_runner.py b/src/basic_memory/indexing/accepted_note_write_runner.py index 3eb6fbc79..b98dae8d8 100644 --- a/src/basic_memory/indexing/accepted_note_write_runner.py +++ b/src/basic_memory/indexing/accepted_note_write_runner.py @@ -21,6 +21,7 @@ RelationGenerationPublication, RelationGenerationStore, SectionGenerationStore, + TemporalGenerationStore, ) from basic_memory.models import Entity, NoteContent from basic_memory.repository import ( @@ -189,6 +190,10 @@ class AcceptedNoteSectionRepository(SectionGenerationStore, Protocol): """Generation-fenced section persistence for accepted note writes.""" +class AcceptedNoteTemporalRepository(TemporalGenerationStore, Protocol): + """Generation-fenced valid-time persistence for accepted note writes.""" + + class AcceptedNoteRelationRepository(RelationGenerationStore, Protocol): """Generation-fenced relation persistence for accepted note writes.""" @@ -221,6 +226,11 @@ def section_repository( project_id: ProjectId, ) -> AcceptedNoteSectionRepository: ... + def temporal_repository( + self, + project_id: ProjectId, + ) -> AcceptedNoteTemporalRepository: ... + def relation_repository( self, project_id: ProjectId, @@ -612,6 +622,7 @@ async def accepted_relation_generation_publication( category=observation.category, context=observation.context, tags=observation.tags, + temporal=observation.temporal, ) for observation in observations ) diff --git a/src/basic_memory/indexing/batch_indexer.py b/src/basic_memory/indexing/batch_indexer.py index b8dd1446e..2cff00399 100644 --- a/src/basic_memory/indexing/batch_indexer.py +++ b/src/basic_memory/indexing/batch_indexer.py @@ -44,6 +44,7 @@ from basic_memory.models import Entity, NoteContent, Relation, RelationSearchRefresh from basic_memory.repository import EntityRepository, ObservationRepository, RelationRepository from basic_memory.repository.note_content_repository import NoteContentRepository +from basic_memory.repository.memory_time_index_repository import MemoryTimeIndexRepository from basic_memory.repository.note_section_repository import NoteSectionRepository from basic_memory.repository.semantic_errors import SemanticDependenciesMissingError from basic_memory.repository.relation_repository import lock_note_content_before_entity_mutation @@ -163,6 +164,7 @@ def __init__( self.relation_repository = relation_repository self.note_content_repository = NoteContentRepository(project_id=project_id) self.section_repository = NoteSectionRepository(project_id=project_id) + self.temporal_repository = MemoryTimeIndexRepository(project_id=project_id) self.search_service = search_service self.file_writer = file_writer self.session_maker = session_maker @@ -170,6 +172,7 @@ def __init__( relation_repository=relation_repository, observation_repository=observation_repository, section_repository=self.section_repository, + temporal_repository=self.temporal_repository, session_maker=session_maker, ) self.relation_resolution = RepositoryRelationResolutionRuntime( @@ -773,6 +776,10 @@ async def _clear_note_only_state(self, session: AsyncSession, entity: Entity) -> relation.to_entity = None await self.observation_repository.delete_by_fields(session, entity_id=entity.id) + # Valid time is asserted by observations, so it retires with them: a resource + # that is no longer Markdown asserts nothing, and no later pass would repair + # rows left addressing observations that have just been deleted. + await self.temporal_repository.delete_by_fields(session, entity_id=entity.id) await self.section_repository.delete_by_fields(session, entity_id=entity.id) await self.relation_repository.delete_by_fields(session, from_id=entity.id) await self.note_content_repository.delete_by_entity_id(session, entity.id) @@ -1092,6 +1099,7 @@ async def _build_prepared_entity( category=observation.category, context=observation.context, tags=observation.tags, + temporal=tuple(observation.temporal), ) for observation in prepared.markdown.observations ) diff --git a/src/basic_memory/indexing/models.py b/src/basic_memory/indexing/models.py index a733fcea0..477cdb5d6 100644 --- a/src/basic_memory/indexing/models.py +++ b/src/basic_memory/indexing/models.py @@ -44,6 +44,7 @@ StorageEtag, normalize_storage_etag, ) +from basic_memory.temporal import TemporalAssertion if TYPE_CHECKING: # pragma: no cover from basic_memory.models import Entity @@ -123,6 +124,9 @@ class IndexedObservation: category: str | None context: str | None tags: list[str] | None + # Authored valid time (SPEC-82). Published as its own projection keyed on the + # observation row this write creates, never as observation columns. + temporal: tuple[TemporalAssertion, ...] = () @dataclass(frozen=True, slots=True) diff --git a/src/basic_memory/indexing/relation_persistence.py b/src/basic_memory/indexing/relation_persistence.py index eb260b5fe..0579f8b17 100644 --- a/src/basic_memory/indexing/relation_persistence.py +++ b/src/basic_memory/indexing/relation_persistence.py @@ -1,7 +1,7 @@ """Publish a note's derived graph under one accepted content generation. -The publication carries the complete observation, section, and relation projections -through one note_content fence lifecycle and one durable retry marker. +The publication carries the complete observation, temporal, section, and relation +projections through one note_content fence lifecycle and one durable retry marker. The graph tables are eventually consistent projections, not part of the accepted write's transaction. Publication runs after the content commit; a stale fence makes every statement @@ -23,6 +23,10 @@ from basic_memory import db from basic_memory.indexing.models import IndexedObservation, IndexedRelation, IndexedSection +from basic_memory.repository.memory_time_index_repository import ( + AcceptedTemporalAssertion, + TemporalGenerationWriteResult, +) from basic_memory.repository.note_section_repository import ( AcceptedSectionWrite, SectionGenerationWriteResult, @@ -37,6 +41,7 @@ RelationGenerationWriteResult, ) from basic_memory.runtime.storage import ProjectId, RuntimeEntityId, RuntimeNoteContentVersion +from basic_memory.schemas.search import SearchItemType class RelationGenerationStore(Protocol): @@ -94,6 +99,19 @@ async def replace_sections_for_generation( ) -> SectionGenerationWriteResult: ... +class TemporalGenerationStore(Protocol): + """Repository operation needed to replace one temporal generation.""" + + async def replace_assertions_for_generation( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + assertions: Sequence[AcceptedTemporalAssertion], + ) -> TemporalGenerationWriteResult: ... + + @dataclass(frozen=True, slots=True) class RelationGenerationPublication: """Derived graph intent authorized by one accepted note-content generation.""" @@ -108,11 +126,12 @@ class RelationGenerationPublication: @dataclass(frozen=True, slots=True) class RelationGenerationPublisher: - """Commit observations, sections, relation chunks, and cleanup under source fences.""" + """Commit observations, valid time, sections, relations, and cleanup under fences.""" relation_repository: RelationGenerationStore observation_repository: ObservationGenerationStore section_repository: SectionGenerationStore + temporal_repository: TemporalGenerationStore session_maker: async_sessionmaker[AsyncSession] async def publish( @@ -163,27 +182,11 @@ async def publish( if not publication.generation_is_current: return False - # Observations fit one statement batch, so their fenced replacement owns one short - # transaction. An empty desired set must still execute to wipe the prior projection. - accepted_observations = tuple( - AcceptedObservationWrite( - content=observation.content, - category=observation.category, - context=observation.context, - tags=observation.tags, - ) - for observation in observations - ) - async with db.scoped_session(self.session_maker) as session: - observation_result = ( - await self.observation_repository.replace_observations_for_generation( - session, - entity_id=entity_id, - generation=generation, - observations=accepted_observations, - ) - ) - if not observation_result.generation_is_current: + if not await self._publish_observations( + entity_id=entity_id, + generation=generation, + observations=observations, + ): return False # Sections mirror the observation projection: one fenced wipe-and-recreate in @@ -234,3 +237,85 @@ async def publish( generation=generation, ) return cleanup.generation_is_current + + async def _publish_observations( + self, + *, + entity_id: int, + generation: int, + observations: Sequence[IndexedObservation], + ) -> bool: + """Replace observations and their authored valid time under one fence. + + Observations fit one statement batch, so their fenced replacement owns one + short transaction. An empty desired set must still execute, for both writes, + to wipe the prior projections. + + Constraint: the temporal projection addresses observations by row id, and + those ids are minted by the insert here. Unlike sections and observations, + which key on the stable entity_id, this write cannot be deferred to a + transaction of its own: a same-generation republish landing between two + commits would wipe and re-mint the observation rows, leaving temporal rows + addressing ids that no longer exist and that no later pass repairs. One + transaction under one held fence keeps a row and its valid time atomic. That + is a narrow exception earned by the id dependency, not a licence to widen the + other statements. + """ + accepted_observations = tuple( + AcceptedObservationWrite( + content=observation.content, + category=observation.category, + context=observation.context, + tags=observation.tags, + temporal=observation.temporal, + ) + for observation in observations + ) + async with db.scoped_session(self.session_maker) as session: + observation_result = ( + await self.observation_repository.replace_observations_for_generation( + session, + entity_id=entity_id, + generation=generation, + observations=accepted_observations, + ) + ) + if not observation_result.generation_is_current: + return False + + temporal_result = await self.temporal_repository.replace_assertions_for_generation( + session, + entity_id=entity_id, + generation=generation, + assertions=_accepted_temporal_assertions( + observations, + observation_result.observation_ids, + ), + ) + return temporal_result.generation_is_current + + +def _accepted_temporal_assertions( + observations: Sequence[IndexedObservation], + observation_ids: Sequence[int], +) -> tuple[AcceptedTemporalAssertion, ...]: + """Pair each authored assertion with the observation row that now carries it. + + Both sequences are in document order, so position is the pairing. A length + mismatch means the observation write returned ids for a different set of rows + than it was given, which would silently attach valid time to the wrong statement. + """ + if len(observation_ids) != len(observations): + raise ValueError( + f"Observation publication returned {len(observation_ids)} row ids " + f"for {len(observations)} observations" + ) + return tuple( + AcceptedTemporalAssertion( + source_type=SearchItemType.OBSERVATION.value, + source_id=observation_id, + assertion=assertion, + ) + for observation, observation_id in zip(observations, observation_ids) + for assertion in observation.temporal + ) diff --git a/src/basic_memory/man/man3/search-notes(3).md b/src/basic_memory/man/man3/search-notes(3).md index d982154ee..41d81a8b5 100644 --- a/src/basic_memory/man/man3/search-notes(3).md +++ b/src/basic_memory/man/man3/search-notes(3).md @@ -25,7 +25,8 @@ search_notes(query=None, project=None, project_id=None, search_type=None, output_format="text", note_types=None, entity_types=None, categories=None, after_date=None, metadata_filters=None, tags=None, status=None, - min_similarity=None) + min_similarity=None, valid_at=None, valid_overlaps=None, + time_kind=None) ``` CLI: @@ -52,6 +53,24 @@ rows), `categories` (observation categories, paired with `metadata_filters` — equality matches against arbitrary frontmatter fields, which is how the manual implements apropos (see [[Manpage]]). +**Valid time** (`valid_at`, `valid_overlaps`, `time_kind`) queries what a note +*says was true*, not when it was last edited. Observations can carry a +qualifier — a range, `- [decision] @effective[2026-06-10,2026-07-27) ...`, or +a point, `- [decision] @effective:2026-07-27 ...` / `- [decision] @2026-07-27 +...` — and these filters match against that authored interval. A point means +the span its precision covers: `@2026` is that year, `@2026-06` that month, +and `@2026-06-10` from that date onward. An unquoted point is one +whitespace-delimited token; a multi-word, relative, or month-only date goes in +double quotes, which end the token at the closing quote: +`@occurred:"June 10, 2026"`, `@occurred:"2 days ago"`, `@"June 2026"`. It is a +separate axis from `after_date`, which keeps filtering last-indexed time. +Bounds follow +PostgreSQL range conventions, calendar dates and instants never convert into +one another, and a source with no qualifier is excluded from any valid-time +query. Because one note can carry several assertions that disagree, these +queries return observation-level results, each carrying the assertion that +matched. + ## PARAMETERS - **query** — search string; optional. Omit it for filter-only searches @@ -65,6 +84,14 @@ which is how the manual implements apropos (see [[Manpage]]). - **tags** — list or comma string, same convention as [[write-note(3)]] - **min_similarity** — float override for vector/hybrid threshold; `0.0` shows everything, `0.8` is high precision +- **valid_at** — date (`2026-07-28`) or RFC 3339 instant + (`2026-07-28T09:00:00Z`) that the authored range must contain; a timestamp + written without an offset is read as UTC (aliases: `as_of`, `valid_on`) +- **valid_overlaps** — range literal the authored range must overlap: + `[2026-06-10,2026-07-27)`, `(,2026-07-27]`, `[2026-06-10,)`. Mutually + exclusive with `valid_at` (aliases: `overlaps`, `valid_during`) +- **time_kind** — kind of valid time: `effective`, `valid`, `occurred`, `due`, + or `mentioned`; usable on its own (alias: `kind`) - **search_all_projects** — opt-in cross-project search; ignored when `project`/`project_id` is given - **page**, **page_size** — pagination (aliases: `page_number`, `limit`, @@ -103,6 +130,14 @@ bm tool search-notes "conflict error" --project manual --page-size 2 - [gotcha] Score semantics differ by mode: FTS rank scores in text mode, similarity scores in hybrid/vector — don't compare across modes #scoring - [gotcha] The CLI takes QUERY positionally; there is no --query flag #cli-parity - [gotcha] search_all_projects is silently ignored when a project is specified #routing +- [gotcha] A valid-time filter excludes every source without a temporal qualifier — an undated note makes no claim about when it holds, so drop the filter to search dated and undated content together #valid-time +- [gotcha] valid_at and valid_overlaps never mix calendar dates with instants: a date query matches only date ranges and an instant query only instant ranges, so `2026-07-27` and `2026-07-27T00:00:00Z` are different questions #valid-time +- [gotcha] A timestamp written without an offset is read as UTC, in an authored qualifier and in a filter alike — same convention as every other naive datetime in Basic Memory #valid-time +- [gotcha] An authored token that does not read as a date is left as ordinary observation content with no warning; only a qualifier the author plainly meant is reported — an unknown kind (`@asserted:2026-06-10`), an unterminated quote, or a date the one-token rule truncated #valid-time +- [gotcha] An unquoted authored point is one whitespace-delimited token: `@occurred:2026-06-10`, `@occurred:03/04/2026` and `@occurred:yesterday` work, but a multi-word date like `@occurred:June 10, 2026` is left as content because nothing can tell where it ends #valid-time +- [gotcha] Double quotes lift the one-token rule and end the point at the closing quote, so `@occurred:"June 10, 2026"`, `@occurred:"2 days ago"` and `@"June 2026"` all file — inside quotes even a month-only or year-only date is taken, since the author delimited it #valid-time +- [gotcha] Only `"` opens a quoted point, never `'`, and an unterminated quote is reported rather than swallowing the rest of the line #valid-time +- [gotcha] `@occurred:03/04/2026` resolves by the `date_order` setting (YMD/DMY read it as 3 April, MDY as 4 March); ISO dates are never re-guessed #valid-time ## SEE ALSO diff --git a/src/basic_memory/markdown/entity_parser.py b/src/basic_memory/markdown/entity_parser.py index 495b9702b..d0caf6fa7 100644 --- a/src/basic_memory/markdown/entity_parser.py +++ b/src/basic_memory/markdown/entity_parser.py @@ -335,6 +335,24 @@ async def parse_markdown_content( entity_content = ( parse(post.content) if parse_semantics else EntityContent(content=post.content) ) + + # The parser reports only a qualifier the author plainly meant: an unknown kind, + # an unterminated quote, or a date the one-token rule truncated. None of those + # reach the index, so this warning is how an author learns the line needs fixing. + # Text that simply is not a date is ordinary content and says nothing here. The + # typed `temporal_error` field carries the same message to programmatic callers; + # this layer adds the path. + # `as_posix()` rather than the Path itself: Basic Memory names files with + # forward slashes everywhere (entity.file_path, permalinks, search rows), so a + # Windows `WindowsPath` rendering `decisions\cache-layer.md` would print a path + # the author cannot find in any other surface. + for observation in entity_content.observations: + if observation.temporal_error: + logger.warning( + f"Temporal qualifier ignored in {file_path.as_posix()}: " + f"{observation.temporal_error}" + ) + # Sections are structural, not semantic: they index the body for range reads, # so the bm_parse_semantics opt-out above must not suppress them. sections = scan_sections(post.content) diff --git a/src/basic_memory/markdown/plugins.py b/src/basic_memory/markdown/plugins.py index af00c0b0d..d88a4a7e2 100644 --- a/src/basic_memory/markdown/plugins.py +++ b/src/basic_memory/markdown/plugins.py @@ -3,6 +3,7 @@ import re from typing import List, Any, Dict +from basic_memory.markdown.temporal_qualifier import parse_temporal_qualifier from basic_memory.utils import normalize_project_reference from markdown_it import MarkdownIt from markdown_it.rules_inline.backticks import backtick @@ -100,6 +101,14 @@ def parse_observation(token: Token) -> Dict[str, Any]: if empty_match: content = empty_match.group(1).strip() + # Parse the temporal qualifier before the (context) rule below. An authored + # `@effective(2026-06-10,2026-07-27)` at end of line ends in ")", so the context + # rule would otherwise claim it and leave `@effective` as the whole observation. + # A qualifier that was not accepted is never peeled, so that line keeps its exact + # text (SPEC-82). + temporal = parse_temporal_qualifier(content) + content = temporal.content + # Parse (context) context = None if content.endswith(")"): @@ -124,6 +133,8 @@ def parse_observation(token: Token) -> Dict[str, Any]: "content": content, "tags": tags if tags else None, "context": context, + "temporal": list(temporal.assertions), + "temporal_error": temporal.error, } diff --git a/src/basic_memory/markdown/schemas.py b/src/basic_memory/markdown/schemas.py index 8788efd24..4f9c30d71 100644 --- a/src/basic_memory/markdown/schemas.py +++ b/src/basic_memory/markdown/schemas.py @@ -6,6 +6,7 @@ from pydantic import BaseModel, Field, model_validator from basic_memory.markdown.sections import MarkdownSection +from basic_memory.temporal import TemporalAssertion class Observation(BaseModel): @@ -15,10 +16,23 @@ class Observation(BaseModel): content: str tags: Optional[List[str]] = None context: Optional[str] = None + # Collection-shaped from day one: the MVP parses at most one qualifier per + # observation, but carrying several later must not be a schema break (SPEC-82). + temporal: List[TemporalAssertion] = [] + # Set for the three reported cases: an unknown kind, an unterminated quote, and an + # unquoted point the one-token rule truncated. Its text stays in `content`, so + # nothing is dropped -- only the derived temporal projection is withheld until the + # author fixes the line. Text that simply is not a date sets nothing here; it is + # ordinary content. + temporal_error: Optional[str] = None @override def __str__(self) -> str: - obs_string = f"- [{self.category}] {self.content}" + # Replaying `source_text` verbatim is what makes parse/serialize a byte-exact + # round trip: `valid_during` holds normalized bounds, the author's text does not. + qualifiers = " ".join(assertion.source_text for assertion in self.temporal) + prefix = f"{qualifiers} " if qualifiers else "" + obs_string = f"- [{self.category}] {prefix}{self.content}" if self.context: obs_string += f" ({self.context})" return obs_string diff --git a/src/basic_memory/markdown/temporal_qualifier.py b/src/basic_memory/markdown/temporal_qualifier.py new file mode 100644 index 000000000..9254288f3 --- /dev/null +++ b/src/basic_memory/markdown/temporal_qualifier.py @@ -0,0 +1,342 @@ +"""Peel SPEC-82 temporal qualifiers off observation content. + +An observation may carry one qualifier immediately after its category and before its +content. Three authored forms exist, and the kind is optional in all of them: + + - [decision] @effective[2026-06-10,2026-07-27) The cache layer will use Redis. + - [decision] @effective:2026-07-27 The cache layer will use Memcached. + - [decision] @effective:"June 10, 2026" The cache layer will use Memcached. + - [decision] @2026-07-27 The cache layer will use Memcached. + +The bracket form carries a range literal and needs no separator, because no kind name +can begin with `[` or `(`. The point forms need the `:` because a date can begin with a +letter (`yesterday`), so nothing else would tell `@occurred:yesterday` from a handle. + +**An unquoted point is one whitespace-delimited token.** dateparser reads far more than +one token -- `June 10, 2026`, `2 days ago`, `2026-06-10 10:00 AM` all resolve, and +`parse_authored_point` accepts them -- but nothing here can tell where such a date ends: +dateparser also reads `June 10, 2026 The` and `2026-06-10 The`, so growing the token +until parsing fails would swallow the author's prose. + +**A quoted point is exactly what the author put between the quotes**, which is how a +multi-word, relative, or month-only date is written: `@occurred:"June 10, 2026"`, +`@occurred:"2 days ago"`, `@"June 2026"`. The closing quote is the token boundary, so +whatever follows it is ordinary content, and a `\\"` inside the value does not end the +token. The scan mirrors `_split_predicate_items` in `mcp/tools/posix_tools.py`, down to +its rule that an unterminated quote is a typo to report rather than a boundary to guess +at -- scanning on to end of line would hand the author's prose to dateparser. Only the +double quote opens the form: an apostrophe is ordinary punctuation, and a scan looking +for its partner would turn `@note:'s` and its like into diagnostics. + +Inside quotes the author delimited the value, so there is nothing to truncate and +dateparser's reading is taken as written. An *unquoted* token is refused in two shapes +that do parse, so a truncated read never becomes a plausible-looking assertion: + +* **A short number.** dateparser reads `1` as January and `3.5` as March 5, but at the + head of a line those are list markers and version numbers. A numeric point must be at + least as wide as a year. +* **A word naming only a month or a year** (`June`, `may`, `v2`). Alone it is usually + prose; as the first token of `June 10, 2026` reading it would file June 2026 and leave + `10, 2026` in the content. A word is taken only when it names a specific day + (`yesterday`, `today`), in whatever language dateparser resolves it. + +Beyond those, one rule decides everything: **if the payload reads as time, the token +becomes a qualifier; if it does not, the token stays ordinary observation content, +silently.** Prose is full of `@` -- email addresses, handles, `@todo:` markers -- and +warning about each one that is not a date would be noise, not help. + +Three things are reported instead, because each one names its own fix: + +* an **unknown kind** (`@asserted:2026-06-10`) -- the payload parses as time and the + author is plainly reaching for this feature, so a short list of valid kinds helps; +* an **unterminated quote** -- the author opened the quoted form and mistyped; +* an unquoted point refused by the guards above **whose line continues with a digit** + (`@occurred:June 10, 2026 ...`) -- the one shape where the token rule silently costs + the author a date they clearly wrote, and the quoted form is what they wanted. + +A refused or unread qualifier is never peeled. Its text stays in the observation +content, so the line indexes exactly as it does today and remains full-text searchable; +only the derived temporal projection is withheld. +""" + +import re +from dataclasses import dataclass + +from basic_memory.temporal import ( + DateOrder, + TemporalAssertion, + TemporalQualifierError, + TemporalRange, + TimeKind, + parse_authored_point, + parse_range_literal, +) + +_KIND_NAMES = frozenset(kind.value for kind in TimeKind) + +# A point with no kind is filed as valid time, the kind this feature is named for: the +# author said when the statement holds without narrowing *how* it holds. +DEFAULT_TIME_KIND = TimeKind.VALID + +_KIND_PATTERN = r"[A-Za-z][A-Za-z0-9_]*" + +# `@[kind]` glued to one balanced bracket group carrying a range literal's comma. The +# lookahead stops `@effective[a,b)x` from half-matching, and the `^` anchor keeps +# `paul@basicmemory.com` and mid-sentence `@handles` out entirely. +_RANGE_QUALIFIER = re.compile(rf"^@({_KIND_PATTERN})?([\[(][^\[\]()]*,[^\[\]()]*[\])])(?=\s|$)") + +# `@[kind:]"` -- the opening of the quoted point. Only the quote is matched here; its +# partner is found by a scan, because a regex cannot honor `\"`. +_QUOTED_POINT_QUALIFIER = re.compile(rf'^@(?:({_KIND_PATTERN}):)?"') + +# `@kind:`. +_KIND_POINT_QUALIFIER = re.compile(rf"^@({_KIND_PATTERN}):(\S+)") + +# `@` -- the point with no kind. Without one there is nothing to +# distinguish a word from a handle, so only digits open the form at all. +_BARE_POINT_QUALIFIER = re.compile(r"^@(\d\S*)") + +_QUOTE = '"' + +# The width of a year, and the shortest numeric token worth reading as one. +_MIN_NUMERIC_POINT_WIDTH = 4 + +# Every diagnostic that a quote would have fixed shows the form rather than describing +# it, so the fix is one copyable edit away. +_QUOTED_EXAMPLE = "June 10, 2026" + + +@dataclass(frozen=True, slots=True) +class ObservationTemporalParse: + """What a qualifier scan found at the head of one observation's content. + + Exactly three shapes exist: a peel (content shortened, one assertion, no error), a + refusal (content untouched, no assertions, an error message naming the fix), and no + qualifier at all (content untouched, nothing found). + """ + + content: str + assertions: tuple[TemporalAssertion, ...] + error: str | None + + +def _no_qualifier(content: str) -> ObservationTemporalParse: + """Leave the line exactly as authored, with nothing to report.""" + return ObservationTemporalParse(content=content, assertions=(), error=None) + + +def _refuse(content: str, reason: str) -> ObservationTemporalParse: + """Keep the line exactly as authored and report why no assertion was derived.""" + return ObservationTemporalParse(content=content, assertions=(), error=reason) + + +@dataclass(frozen=True, slots=True) +class _ReadQualifier: + """One token that read as time: how much of the line it spans, and what it says.""" + + token: str + end: int + kind_name: str | None + valid_during: TemporalRange + + +@dataclass(frozen=True, slots=True) +class _Refusal: + """A qualifier the author plainly meant, reported instead of silently kept.""" + + reason: str + + +@dataclass(frozen=True, slots=True) +class _PointToken: + """Where a point form ends, and the text handed to the date reader. + + `quoted` is what separates the two point forms once the boundary is found: the + author delimited a quoted value, so the truncation guards below have nothing to + guard against. + """ + + point: str + end: int + kind_name: str | None + quoted: bool + + +# What a scan of the head of one line can find: a qualifier, a reportable mistake, or +# nothing at all. +type _QualifierScan = _ReadQualifier | _Refusal | None + + +def _read_range_qualifier(content: str) -> _ReadQualifier | None: + """Match the bracket form and parse its literal, or report no usable qualifier.""" + match = _RANGE_QUALIFIER.match(content) + if match is None: + return None + try: + valid_during = parse_range_literal(match.group(2)) + except TemporalQualifierError: + # A literal we cannot read is not a qualifier. Saying *how* it is malformed + # would be a diagnostic about how someone wrote a date, which this feature + # deliberately does not issue. + return None + return _ReadQualifier(match.group(0), match.end(), match.group(1), valid_during) + + +def _scan_quoted_point(content: str, opened_at: int) -> tuple[str, int] | None: + """Read a quoted payload from `opened_at` to its closing quote. + + One pass with a backslash escape, the same scan `_split_predicate_items` uses for + find's predicate values: the delimiter rather than whitespace ends the token, and an + escaped quote belongs to the value. Returns the value and the index just past the + closing quote, or None when the quote never closed. + """ + value: list[str] = [] + escaped = False + for index in range(opened_at, len(content)): + char = content[index] + if escaped: + value.append(char) + escaped = False + elif char == "\\": + escaped = True + elif char == _QUOTE: + return "".join(value), index + 1 + else: + value.append(char) + return None + + +def _locate_point(content: str) -> _PointToken | _Refusal | None: + """Find a point form at the head of the line and delimit the date it carries.""" + quoted = _QUOTED_POINT_QUALIFIER.match(content) + if quoted is not None: + scanned = _scan_quoted_point(content, quoted.end()) + if scanned is None: + # Trigger: the author opened the quoted form and never closed it. + # Why: every other reading of the line is a guess -- taking the rest of it + # would hand prose to dateparser, and dropping the quote would put the + # truncation this form exists to prevent right back. + # Outcome: the line keeps its text and the author is told which keystroke + # is missing. + return _Refusal( + f"unterminated quote in temporal qualifier {quoted.group(0)!r}; " + f'close it, as {quoted.group(0)}{_QUOTED_EXAMPLE}"' + ) + point, end = scanned + return _PointToken(point=point, end=end, kind_name=quoted.group(1), quoted=True) + + named = _KIND_POINT_QUALIFIER.match(content) + bare = None if named is not None else _BARE_POINT_QUALIFIER.match(content) + match = named or bare + if match is None: + return None + return _PointToken( + point=match.group(2) if named is not None else match.group(1), + end=match.end(), + kind_name=match.group(1) if named is not None else None, + quoted=False, + ) + + +def _truncation_reason(point: str, valid_during: TemporalRange) -> str | None: + """Why an unquoted point is too coarse to file, or None when it names a day. + + The two shapes named here both parse, which is exactly why they need refusing -- + see the module docstring for what each one costs if it is read. The wording is the + diagnostic's, so the reason a token was refused and the reason it *is* refused stay + the same sentence. + + A bounded span is how a coarse point announces itself: `parse_authored_point` closes + a year or a month at its successor and leaves a day or a moment open, so + `upper is None` *is* "this names a specific day". The one period with no successor + to close at -- December 9999 -- is left open too, and so reads here as a day; no word + resolves to it, so the guard never sees that shape. + """ + if point[0].isdigit(): + return None if len(point) >= _MIN_NUMERIC_POINT_WIDTH else "is narrower than a year" + return None if valid_during.upper is None else "names only a month or a year" + + +def _truncated_point_refusal(content: str, token: _PointToken, reason: str) -> _Refusal | None: + """Report a refused token that reads as the first word of a longer date. + + Trigger: a known (or omitted) kind, and content after the refused token starting + with a digit. + Why: `@occurred:June 10, 2026` is the one shape where the one-token rule silently + costs the author a date they clearly wrote, and the digit is the only signal that + the date kept going. Prose after the token (`@occurred:June the cat sat`) is just + prose, and an unknown kind (`@vol:2 3 pages`) is an ordinary `@word:` marker; + diagnosing either would fire all over an ordinary vault. + Outcome: one sentence naming the quoted form that files the whole date. Otherwise + the token stays ordinary content, silently, exactly as it did before quoting. + """ + if token.kind_name is not None and token.kind_name not in _KIND_NAMES: + return None + rest = content[token.end :].lstrip() + if not rest or not rest[0].isdigit(): + return None + prefix = content[: token.end - len(token.point)] + return _Refusal( + f"temporal point {content[: token.end]!r} {reason}; " + f'quote the whole date to file it, as {prefix}"{_QUOTED_EXAMPLE}"' + ) + + +def _read_point_qualifier(content: str, date_order: DateOrder | None) -> _QualifierScan: + """Match any point form and read its date, or report why nothing was filed.""" + located = _locate_point(content) + if located is None or isinstance(located, _Refusal): + return located + + # Deferred, following utils.ensure_timezone_aware: the markdown parser is a + # low-level module that many entrypoints import, and pulling the config stack in at + # import time couples parsing to configuration load order for no benefit. Resolved + # here rather than at the top of the scan so only a token that already looks like a + # qualifier pays for reading the config -- or for loading dateparser. + from basic_memory.config import ConfigManager + + order = date_order if date_order is not None else ConfigManager().config.date_order + valid_during = parse_authored_point(located.point, date_order=order) + if valid_during is None: + return None + + # Quotes are the author's own delimiters, so a quoted value cannot be the truncated + # head of a longer date and the guards do not apply to it. + reason = None if located.quoted else _truncation_reason(located.point, valid_during) + if reason is not None: + return _truncated_point_refusal(content, located, reason) + return _ReadQualifier(content[: located.end], located.end, located.kind_name, valid_during) + + +def parse_temporal_qualifier( + content: str, *, date_order: DateOrder | None = None +) -> ObservationTemporalParse: + """Split a leading temporal qualifier off observation content. + + The MVP reads at most one qualifier per observation, but the result is a collection + so supporting several later is not a schema break. `date_order` defaults to the + configured `date_order`; tests and callers that already hold the config pass it. + """ + read = _read_range_qualifier(content) or _read_point_qualifier(content, date_order) + if isinstance(read, _Refusal): + return _refuse(content, read.reason) + if read is None: + return _no_qualifier(content) + + kind_name = read.kind_name + if kind_name is not None and kind_name not in _KIND_NAMES: + known = ", ".join(sorted(_KIND_NAMES)) + return _refuse(content, f"unknown temporal kind {kind_name!r} in {read.token!r} ({known})") + + remainder = content[read.end :].strip() + if not remainder: + # A qualifier with nothing to qualify would leave an empty observation, which + # the plugin drops outright. Keep the line whole instead. + return _no_qualifier(content) + + assertion = TemporalAssertion( + time_kind=TimeKind(kind_name) if kind_name is not None else DEFAULT_TIME_KIND, + valid_during=read.valid_during, + source_text=read.token, + ) + return ObservationTemporalParse(content=remainder, assertions=(assertion,), error=None) diff --git a/src/basic_memory/mcp/clients/search.py b/src/basic_memory/mcp/clients/search.py index 0d04fda2c..08886178f 100644 --- a/src/basic_memory/mcp/clients/search.py +++ b/src/basic_memory/mcp/clients/search.py @@ -14,6 +14,10 @@ # so each method defers the import to call time instead (#886). from basic_memory.schemas.search import SearchResponse, SearchRetrievalMode +# The valid-time fields SearchQuery carries. Named here so the skew check below stays +# in step with the schema without importing the model's internals. +_TEMPORAL_QUERY_FIELDS = ("valid_at", "valid_overlaps", "time_kind") + class SearchClient: """Typed client for search operations. @@ -59,6 +63,7 @@ async def search( Raises: ToolError: If the request fails + ValueError: If a requested valid-time filter was not applied by the server """ from basic_memory.mcp.tools.utils import call_query @@ -87,4 +92,21 @@ async def search( retrieval_mode = query.get("retrieval_mode", SearchRetrievalMode.FTS) payload["total_is_exact"] = retrieval_mode == SearchRetrievalMode.FTS + # Trigger: this request carried a valid-time filter but the response does not + # confirm the server ran it. + # Why: SearchQuery ignores unknown fields, so a server predating SPEC-82 accepts + # the request and returns results that look filtered. A valid-time query + # excludes undated sources; unfiltered results include them, and the caller + # would have no way to tell. + # Outcome: fail loudly instead of returning a wrong answer that reads as right. + if any(query.get(field) for field in _TEMPORAL_QUERY_FIELDS) and ( + payload.get("temporal_applied") is not True + ): + raise ValueError( + "The search API did not apply the requested valid-time filter " + "(no temporal_applied confirmation in the response). The server is " + "likely older than this client; upgrade it or drop valid_at / " + "valid_overlaps / time_kind from the query." + ) + return SearchResponse.model_validate(payload) diff --git a/src/basic_memory/mcp/tools/search.py b/src/basic_memory/mcp/tools/search.py index a3db00834..a36a2ebc7 100644 --- a/src/basic_memory/mcp/tools/search.py +++ b/src/basic_memory/mcp/tools/search.py @@ -39,6 +39,7 @@ SearchResult, SearchRetrievalMode, ) +from basic_memory.temporal import TemporalQualifierError, parse_temporal_filter _SERVICE_UNAVAILABLE_HEADING = "# Search Failed - Service Temporarily Unavailable" @@ -395,6 +396,14 @@ def _format_search_markdown( parts.append(f"- score: {r.score:.4f}") if r.matched_chunk: parts.append(f"- match: {r.matched_chunk[:200]}") + # Name the kind and the units. A bare "2026-06-10" here would read as an edit + # date; "effective valid time ... (date)" says which time this is and that it + # is a calendar date carrying no timezone. + for assertion in r.temporal or []: + parts.append( + f"- {assertion.kind} valid time: {assertion.valid_during.literal} " + f"({assertion.valid_during.axis})" + ) parts.append("") # --- Footer with pagination --- @@ -575,11 +584,21 @@ async def _search_all_projects( tags: list[str] | None, status: str | None, min_similarity: float | None, + valid_at: str | None, + valid_overlaps: str | None, + time_kind: str | None, context: Context | None, ) -> dict[str, Any] | str: """Search every accessible project when the caller explicitly opts in.""" requested_page = max(page, 1) requested_page_size = max(page_size, 1) + # Each per-project call runs through search_notes -> SearchClient, which refuses a + # response that does not confirm the filter ran. So a project either honored the + # valid-time filter or was dropped with a warning below; the merged answer never + # silently mixes filtered and unfiltered rows. The filter itself is already known to + # be well formed -- search_notes parses it before reaching here -- which is what + # makes "dropped with a warning" mean an unavailable project and nothing else. + temporal_requested = bool(valid_at or valid_overlaps or time_kind) project_refs = await _load_search_project_refs(context=context) if not project_refs: response = SearchResponse( @@ -589,6 +608,7 @@ async def _search_all_projects( total=0, total_is_exact=True, has_more=False, + temporal_applied=True if temporal_requested else None, ) if output_format == "json": return response.model_dump(mode="json", exclude_none=True) @@ -636,6 +656,9 @@ async def _search_all_projects( tags=tags, status=status, min_similarity=min_similarity, + valid_at=valid_at, + valid_overlaps=valid_overlaps, + time_kind=time_kind, search_all_projects=False, context=context, ) @@ -679,6 +702,7 @@ async def _search_all_projects( "total": total, "total_is_exact": total_is_exact, "has_more": any_project_has_more or total > end or len(sorted_results) > end, + "temporal_applied": True if temporal_requested else None, } ) @@ -789,6 +813,41 @@ async def search_notes( validation_alias=AliasChoices("min_similarity", "threshold", "similarity_threshold"), ), ] = None, + # --- Valid-time filters (SPEC-82) --- + # A different axis from after_date: these ask what a note SAYS was true, not when + # the note was last touched. Appended at the end of the signature so no existing + # positional caller shifts. + valid_at: Annotated[ + Optional[str], + Field( + default=None, + validation_alias=AliasChoices("valid_at", "as_of", "valid_on"), + ), + "Return only sources whose authored valid range CONTAINS this date " + "('2026-07-28') or RFC 3339 instant ('2026-07-28T09:00:00Z'; a timestamp " + "with no offset is read as UTC). Sources with no temporal qualifier are " + "excluded.", + ] = None, + valid_overlaps: Annotated[ + Optional[str], + Field( + default=None, + validation_alias=AliasChoices("valid_overlaps", "overlaps", "valid_during"), + ), + "Return only sources whose authored valid range OVERLAPS this range literal, " + "written PostgreSQL-style: '[2026-06-10,2026-07-27)', '(,2026-07-27]', " + "'[2026-06-10,)'. Mutually exclusive with valid_at.", + ] = None, + time_kind: Annotated[ + Optional[str], + Field( + default=None, + validation_alias=AliasChoices("time_kind", "kind"), + ), + "Narrow valid-time matching to one authored kind of time: 'effective', " + "'valid', 'occurred', 'due', or 'mentioned'. Usable on its own to find every " + "source carrying an assertion of that kind.", + ] = None, context: Context | None = None, ) -> dict[str, Any] | str: """Search across all content in the knowledge base with comprehensive syntax support. @@ -866,6 +925,59 @@ async def search_notes( `tags` and `status` are shorthand for metadata_filters. If the same key exists in metadata_filters, that value wins. + ### Valid-Time Filters (what a note says was true) + Notes can state when a fact holds, by writing a qualifier on an observation: + + - [decision] @effective[2026-06-10,2026-07-27) The cache layer will use Redis. + - [decision] @effective:2026-07-27 The cache layer will use Memcached. + + The bracket form is an explicit range; the `@kind:date` form is a point, meaning + the span its precision covers — `@2026` that year, `@2026-06` that month, and + `@2026-06-10` from that date onward. The kind may be omitted (`@2026-07-27`), + which files the assertion as `valid` time; a point with no kind has to start + with a digit and be at least as wide as a year, so `@v2` and `@may` stay prose. + + An unquoted point is **one whitespace-delimited token**, because nothing can tell + where a multi-word date ends. Slash dates (`@occurred:03/04/2026`, read by the + `date_order` setting) and single-word relative dates (`@occurred:yesterday`) work + as they are; anything longer goes in double quotes, which move the token boundary + to the closing quote: + + - [decision] @occurred:"June 10, 2026" The cutover ran. + - [decision] @occurred:"2 days ago" The cutover ran. + - [decision] @occurred:"June 2026" The cutover ran. + - [decision] @"June 10, 2026" The cutover ran. + + Whatever is inside the quotes is read as the date, month-only and relative forms + included, and whatever follows the closing quote is ordinary content. An unreadable + token is left as content, never half-read. + + These filters query that authored time, which is a different axis from `after_date` + (last-indexed time) — `after_date` is never reinterpreted as valid time. + - `search_notes("cache layer", kind="effective", valid_at="2026-07-28")` + - Returns the Memcached decision; the Redis decision expired at the cutover. + - `search_notes("cache layer", kind="effective", valid_at="2026-07-01")` + - Returns the Redis decision; Memcached is not yet effective. + - `search_notes("cache layer", kind="effective", valid_overlaps="[2026-06-01,2026-08-01)")` + - Returns both, since each overlaps that window. + - `search_notes("cache layer")` with no valid-time filter + - Both compete under ordinary relevance, exactly as before. + + **Sources with no temporal qualifier are excluded from any valid-time query.** + An undated note makes no claim about when it holds, so it cannot answer "what was + true on this date". Drop the valid-time filter to search dated and undated content + together. + + Because a single note can carry several assertions that disagree (as above), these + queries return observation-level results by default rather than whole notes, and + each result carries the assertion that matched so the answer can explain itself. + + Bounds follow PostgreSQL range conventions: `[` / `]` include an endpoint, `(` / `)` + exclude it, and an omitted side is unbounded. Calendar dates (`2026-07-27`) and + instants (`2026-07-27T16:42:00Z`) are separate axes that never convert into each + other: a date query matches only date ranges, an instant query only instant ranges. + An instant written without an offset is read as UTC. + ### Advanced Pattern Examples - `search_notes("project AND (meeting OR discussion)", project="work-project")` - Complex boolean logic - `search_notes('"exact phrase" AND keyword', project="research")` - Combine phrase and keyword search @@ -906,6 +1018,14 @@ async def search_notes( min_similarity: Optional float to override the global semantic_min_similarity threshold for this query. E.g., 0.0 to see all vector results, or 0.8 for high precision. Only applies to vector and hybrid search types. + valid_at: Optional date ("2026-07-28") or RFC 3339 instant ("2026-07-28T09:00:00Z"; + a timestamp with no offset is read as UTC). Returns sources whose authored + valid range contains it. Sources with no temporal qualifier are excluded. + valid_overlaps: Optional PostgreSQL-style range literal ("[2026-06-10,2026-07-27)", + "(,2026-07-27]", "[2026-06-10,)"). Returns sources whose authored valid range + overlaps it. Mutually exclusive with valid_at; also excludes undated sources. + time_kind: Optional kind of valid time to narrow to: "effective", "valid", + "occurred", "due", or "mentioned". Valid on its own. context: Optional FastMCP context for performance caching. Returns: @@ -990,6 +1110,28 @@ async def search_notes( if page_size < 1: raise ValueError(f"page_size must be >= 1, got {page_size}") + # Trigger: both valid-time forms supplied. + # Why: SearchQuery rejects the pair too, but the tool assigns its fields after + # construction, so that validator never runs on this path — the caller would + # otherwise learn about it as an opaque 422 from the API. + # Outcome: one clear error naming the two mutually exclusive parameters. + if valid_at and valid_overlaps: + raise ValueError("Use either valid_at (containment) or valid_overlaps (overlap), not both.") + + # Trigger: any valid-time filter string is supplied. + # Why: these strings are parsed server-side, so a typo comes back as a 400 that the + # fan-out below cannot tell from a project being unavailable -- it logs the + # project, skips it, and after every project is skipped reports an empty result + # that still claims the filter ran. A malformed filter would read as "no matches" + # instead of as an error. This is the only layer that can tell a client mistake + # from a per-project availability failure, and it shares the parser the search + # service uses so the two can never disagree about what is well formed. + # Outcome: one error naming the bad value, before any project is searched. + try: + parse_temporal_filter(valid_at=valid_at, valid_overlaps=valid_overlaps, time_kind=time_kind) + except TemporalQualifierError as exc: + raise ValueError(f"Invalid valid-time filter: {exc}") from exc + # Trigger: list params arrived via a direct function call instead of the MCP layer. # Why: the BeforeValidator annotations only run through MCP/Pydantic validation; direct # callers (e.g. `bm tool search-notes --type note,task` in cli/commands/tool.py, @@ -1065,6 +1207,9 @@ async def search_notes( tags=tags, status=status, min_similarity=min_similarity, + valid_at=valid_at, + valid_overlaps=valid_overlaps, + time_kind=time_kind, context=context, ) return all_projects_result @@ -1092,9 +1237,13 @@ async def search_notes( or entity_types or categories or after_date + or valid_at + or valid_overlaps + or time_kind ), has_tags_filter=bool(tags), has_status_filter=bool(status), + has_temporal_filter=bool(valid_at or valid_overlaps or time_kind), ): async with get_project_client(project, context=context, project_id=project_id) as ( client, @@ -1175,6 +1324,12 @@ async def search_notes( search_query.status = status if min_similarity is not None: search_query.min_similarity = min_similarity + if valid_at: + search_query.valid_at = valid_at + if valid_overlaps: + search_query.valid_overlaps = valid_overlaps + if time_kind: + search_query.time_kind = time_kind # Reject searches with no criteria at all if search_query.no_criteria(): @@ -1182,7 +1337,7 @@ async def search_notes( "# No Search Criteria\n\n" "Please provide at least one of: `query`, `metadata_filters`, " "`tags`, `status`, `note_types`, `entity_types`, `categories`, " - "or `after_date`." + "`after_date`, `valid_at`, `valid_overlaps`, or `time_kind`." ) # Default to entity-level results to avoid returning individual @@ -1190,14 +1345,17 @@ async def search_notes( # Applied after no_criteria() so that the implicit default doesn't # mask a truly empty search request. if not search_query.entity_types: - # Trigger: a category filter was supplied without an explicit - # entity_types. - # Why: categories only exist on observations — defaulting to "entity" - # (whose rows have NULL category) would AND a category filter against - # entity rows and return nothing, defeating a category-only search. + # Trigger: a category or valid-time filter was supplied without an + # explicit entity_types. + # Why: both only exist on observations — categories live on observation + # rows, and temporal assertions are projected against an + # observation's (type, id). Defaulting to "entity" would AND either + # filter against entity rows and return nothing, defeating the + # whole query. # Outcome: scope the implicit default to observations so - # search_notes(categories=[...]) returns the matching bullets. - if search_query.categories: + # search_notes(categories=[...]) and search_notes(valid_at=...) + # return the matching bullets. + if search_query.categories or search_query.has_temporal_filter(): search_query.entity_types = [SearchItemType("observation")] else: search_query.entity_types = [SearchItemType("entity")] diff --git a/src/basic_memory/models/__init__.py b/src/basic_memory/models/__init__.py index 6164aab74..f0377d8a3 100644 --- a/src/basic_memory/models/__init__.py +++ b/src/basic_memory/models/__init__.py @@ -4,6 +4,7 @@ from basic_memory.models.base import Base from basic_memory.models.knowledge import ( Entity, + MemoryTimeIndex, NoteContent, NoteFileVacate, NoteSection, @@ -17,6 +18,7 @@ "Base", "AcceptedProjectNoteChange", "Entity", + "MemoryTimeIndex", "NoteContent", "NoteFileVacate", "NoteSection", diff --git a/src/basic_memory/models/knowledge.py b/src/basic_memory/models/knowledge.py index f6693fc56..2f79c6295 100644 --- a/src/basic_memory/models/knowledge.py +++ b/src/basic_memory/models/knowledge.py @@ -8,6 +8,7 @@ from sqlalchemy import ( BigInteger, + Boolean, CheckConstraint, Integer, String, @@ -18,6 +19,7 @@ Index, JSON, Float, + false, text, ) from sqlalchemy.orm import Mapped, mapped_column, relationship, validates @@ -132,6 +134,9 @@ class Entity(Base): uselist=False, ) sections = relationship("NoteSection", back_populates="entity", cascade="all, delete-orphan") + time_assertions = relationship( + "MemoryTimeIndex", back_populates="entity", cascade="all, delete-orphan" + ) @validates("created_at", "updated_at") def _normalize_semantic_timestamp(self, attribute_name: str, value: datetime) -> datetime: @@ -334,6 +339,45 @@ def __repr__(self) -> str: # pragma: no cover ) +def observation_permalink_tail(category: str | None, content: str) -> str: + """The part of an observation's permalink that distinguishes it within its note. + + This is the single definition of what makes two observations of one note share an + address, and it is deliberately shared with the writer that assigns + `Observation.duplicate_index`: an ordinal only disambiguates if it is counted over + exactly the identity the permalink is built from. Computing the two separately is the + defect this function exists to prevent -- rebuilding the permalink format inline is + what diverged from the search index for long observations (#929). + + Note what is *not* here. A qualifier (`@effective[...]`) and a `(context)` are peeled + off the line before the observation is stored, so neither reaches this string, and two + observations that differ only in one of them arrive identical. That is not an oversight + to correct by stuffing them back in: the peel is the feature, and valid time is its own + projection rather than an observation column. The note still distinguishes such lines, + so the *address* must too, which is what the ordinal counted over this tail supplies. + + Slug aliasing is why the count keys on this generated text rather than on the raw + values: `Foo Bar` and `foo-bar` are different content that generate one permalink, so + an ordinal counted over raw content would leave them colliding. + + Content is truncated to 200 chars to stay under PostgreSQL's btree index limit of + 2704 bytes. + """ + if len(content) > 200: + # Trigger: content exceeds the 200-char budget imposed by PostgreSQL's + # 2704-byte btree index row limit, so the permalink can only carry a prefix. + # Why: two distinct observations with the same category and an identical + # 200-char prefix would collide on the same synthetic permalink, and the + # search index (permalink-keyed upsert) silently drops the second one. + # Outcome: a short stable digest of the FULL content disambiguates + # truncated permalinks while staying well under the index limit. + digest = hashlib.sha256(content.encode("utf-8")).hexdigest()[:12] + content_for_permalink = f"{content[:200]}-{digest}" + else: + content_for_permalink = content + return generate_permalink(f"observations/{category}/{content_for_permalink}") + + class Observation(Base): """An observation about an entity. @@ -355,6 +399,11 @@ class Observation(Base): tags: Mapped[Optional[list[str]]] = mapped_column( JSON, nullable=True, default=list, server_default="[]" ) + # Which of the note's same-identity observations this one is, in document order. + # See `observation_permalink_tail` for why an ordinal is needed at all and why it is + # stored rather than derived: `permalink` is read on *detached* instances, long after + # the session that could have looked at this row's siblings has closed. + duplicate_index: Mapped[int] = mapped_column(Integer, default=0, server_default=text("0")) # Relationships entity = relationship("Entity", back_populates="observations") @@ -366,30 +415,129 @@ def permalink(self) -> str: We can construct these because observations are always defined in and owned by a single entity. - Content is truncated to 200 chars to stay under PostgreSQL's - btree index limit of 2704 bytes. + `duplicate_index` is what keeps the address faithful when one note says the same + thing twice. It is 0 for the first observation carrying a given identity, so the + overwhelming majority of permalinks are byte-identical to what they have always + been; only the second and later twins gain a trailing ordinal. """ - if len(self.content) > 200: - # Trigger: content exceeds the 200-char budget imposed by PostgreSQL's - # 2704-byte btree index row limit, so the permalink can only carry a prefix. - # Why: two distinct observations with the same category and an identical - # 200-char prefix would collide on the same synthetic permalink, and the - # search index (permalink-keyed upsert) silently drops the second one. - # Outcome: a short stable digest of the FULL content disambiguates - # truncated permalinks while staying well under the index limit. - digest = hashlib.sha256(self.content.encode("utf-8")).hexdigest()[:12] - content_for_permalink = f"{self.content[:200]}-{digest}" - else: - content_for_permalink = self.content - return generate_permalink( - f"{self.entity.permalink}/observations/{self.category}/{content_for_permalink}" + base = generate_permalink( + f"{self.entity.permalink}/{observation_permalink_tail(self.category, self.content)}" ) + if not self.duplicate_index: + return base + return f"{base}/{self.duplicate_index}" @override def __repr__(self) -> str: # pragma: no cover return f"Observation(id={self.id}, entity_id={self.entity_id}, content='{self.content}')" +class MemoryTimeIndex(Base): + """One authored temporal assertion, projected into queryable scalar columns. + + A note can say *when a statement is true of the world*, not merely when the file + was edited:: + + - [decision] @effective[2026-06-10,2026-07-27) The cache layer will use Redis. + + That qualifier is canonical markdown. This table is its derived projection, + rebuilt under the note_content generation fence on every (re)index and removed + with the entity, exactly like observations and sections (SPEC-82). It is never a + second source of temporal truth: reindexing from the markdown reproduces it. + + The table is generic on purpose. ``source_type``/``source_id`` address whatever + carries the assertion -- observations in this MVP -- and match the ``(type, id)`` + pair of the corresponding search row, which is what lets a valid-time filter narrow + search results to the individual observation that was in force. ``source_id`` + deliberately carries no foreign key: it points into a different table per + ``source_type``. Lifecycle is carried instead by ``entity_id``'s cascade plus the + fenced replace, the same two mechanisms note_section relies on. + + Bounds are stored as canonical fixed-width text rather than DATE/TIMESTAMP columns: + + * A date bound is a calendar date and must never acquire a time of day or a + timezone. SQLAlchemy's SQLite ``DateTime`` silently discards an offset, storing + the wrong instant -- exactly the false precision the spec forbids. + * ``basic_memory.temporal`` canonicalizes every bound to a fixed-width form + (``YYYY-MM-DD``; ``YYYY-MM-DDTHH:MM:SS.ffffffZ`` in UTC), so byte-lexicographic + order *is* chronological order and one identical SQL predicate serves both + dialects. + + Native PostgreSQL ``daterange``/``tstzrange`` columns stay available as a later + addition: they would be generated from these columns, which remain the portable + source of truth. + """ + + __tablename__ = "memory_time_index" + __table_args__ = ( + # The valid-time predicate selects (source_type, source_id) after filtering on + # project, kind, and axis, so this index both drives the scan and covers its + # projection. project_id leads it, which is why the column carries no separate + # index of its own the way sibling projection tables do. + Index( + "ix_memory_time_index_lookup", + "project_id", + "time_kind", + "range_axis", + "source_type", + "source_id", + ), + # Fenced replace deletes by entity_id, and the cascade follows the same column. + Index("ix_memory_time_index_entity_id", "entity_id"), + CheckConstraint( + "range_axis IN ('date', 'instant')", + name="ck_memory_time_index_range_axis", + ), + # The empty range has no endpoints at all; representing it with bounds would + # make two rows describe the same interval two different ways. + CheckConstraint( + "NOT is_empty OR (lower_value IS NULL AND upper_value IS NULL)", + name="ck_memory_time_index_empty_has_no_bounds", + ), + # PostgreSQL's rule: an unbounded side cannot be inclusive, because there is no + # endpoint to include. Enforcing it here keeps the query predicates from having + # to defend against a bound state the domain value cannot produce. + CheckConstraint( + "(lower_value IS NOT NULL OR NOT lower_inclusive) " + "AND (upper_value IS NOT NULL OR NOT upper_inclusive)", + name="ck_memory_time_index_unbounded_is_exclusive", + ), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) # pyright: ignore [reportIncompatibleVariableOverride] + project_id: Mapped[int] = mapped_column(Integer, ForeignKey("project.id")) + entity_id: Mapped[int] = mapped_column(Integer, ForeignKey("entity.id", ondelete="CASCADE")) + # Addresses the row that carried the qualifier, and equals the search row's + # (type, id) pair. No FK: the target table varies with source_type. + source_type: Mapped[str] = mapped_column(String(32)) + source_id: Mapped[int] = mapped_column(Integer) + time_kind: Mapped[str] = mapped_column(String(32)) + range_axis: Mapped[str] = mapped_column(String(16)) + # Canonical lexical bounds; NULL means unbounded on that side. + lower_value: Mapped[Optional[str]] = mapped_column(String(32), nullable=True) + upper_value: Mapped[Optional[str]] = mapped_column(String(32), nullable=True) + lower_inclusive: Mapped[bool] = mapped_column(Boolean) + upper_inclusive: Mapped[bool] = mapped_column(Boolean) + is_empty: Mapped[bool] = mapped_column(Boolean, default=False, server_default=false()) + extractor: Mapped[str] = mapped_column(String(32)) + # The qualifier exactly as authored, so a result can explain itself in the + # author's own precision rather than in the canonical form. + source_text: Mapped[str] = mapped_column(Text) + # `metadata` is reserved on the declarative base, so the column follows + # Entity.entity_metadata's naming convention for the same reason. + assertion_metadata: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON, nullable=True) + + entity = relationship("Entity", back_populates="time_assertions") + + @override + def __repr__(self) -> str: # pragma: no cover + return ( + f"MemoryTimeIndex(id={self.id}, entity_id={self.entity_id}, " + f"source={self.source_type}:{self.source_id}, kind='{self.time_kind}', " + f"range='{self.source_text}')" + ) + + class Relation(Base): """A directed relation between two entities.""" diff --git a/src/basic_memory/repository/__init__.py b/src/basic_memory/repository/__init__.py index 090e59e76..6280e34d5 100644 --- a/src/basic_memory/repository/__init__.py +++ b/src/basic_memory/repository/__init__.py @@ -1,4 +1,8 @@ from .entity_repository import EntityRepository +from .memory_time_index_repository import ( + AcceptedTemporalAssertion, + MemoryTimeIndexRepository, +) from .note_content_repository import ( AcceptedNoteContentWrite, NoteContentRepository, @@ -11,6 +15,8 @@ __all__ = [ "EntityRepository", + "AcceptedTemporalAssertion", + "MemoryTimeIndexRepository", "AcceptedNoteContentWrite", "NoteContentRepository", "NoteContentVersionConflict", diff --git a/src/basic_memory/repository/accepted_note_repositories.py b/src/basic_memory/repository/accepted_note_repositories.py index a219b50ac..35b55fccb 100644 --- a/src/basic_memory/repository/accepted_note_repositories.py +++ b/src/basic_memory/repository/accepted_note_repositories.py @@ -4,6 +4,7 @@ from dataclasses import dataclass from basic_memory.repository import ( + MemoryTimeIndexRepository, NoteContentRepository, NoteSectionRepository, ObservationRepository, @@ -51,5 +52,8 @@ def observation_repository(self, project_id: ProjectId) -> ObservationRepository def section_repository(self, project_id: ProjectId) -> NoteSectionRepository: return NoteSectionRepository(project_id=project_id) + def temporal_repository(self, project_id: ProjectId) -> MemoryTimeIndexRepository: + return MemoryTimeIndexRepository(project_id=project_id) + def relation_repository(self, project_id: ProjectId) -> RelationRepository: return RelationRepository(project_id=project_id) diff --git a/src/basic_memory/repository/memory_time_index_repository.py b/src/basic_memory/repository/memory_time_index_repository.py new file mode 100644 index 000000000..4e359a1d1 --- /dev/null +++ b/src/basic_memory/repository/memory_time_index_repository.py @@ -0,0 +1,147 @@ +"""Repository for managing MemoryTimeIndex rows.""" + +from collections import defaultdict +from dataclasses import dataclass +from typing import Iterable, Sequence + +from sqlalchemy.ext.asyncio import AsyncSession + +from basic_memory.models import MemoryTimeIndex +from basic_memory.repository.relation_repository import current_relation_generation_statement +from basic_memory.repository.repository import SELECT_BY_IDS_CHUNK_SIZE, Repository +from basic_memory.temporal import TemporalAssertion + + +@dataclass(frozen=True, slots=True) +class AcceptedTemporalAssertion: + """One authored assertion paired with the persisted row that carried it. + + The parser cannot supply `source_type`/`source_id`: it reads markdown, where the + projection's row identities do not exist yet. Publication mints them and pairs + them here. + """ + + source_type: str + source_id: int + assertion: TemporalAssertion + + +@dataclass(frozen=True, slots=True) +class TemporalGenerationWriteResult: + """Whether a guarded temporal replacement still owned its source generation.""" + + generation_is_current: bool + + +def _projection_row( + accepted: AcceptedTemporalAssertion, + *, + project_id: int, + entity_id: int, +) -> MemoryTimeIndex: + """Flatten one assertion into the portable scalar columns the table stores.""" + valid_during = accepted.assertion.valid_during + return MemoryTimeIndex( + project_id=project_id, + entity_id=entity_id, + source_type=accepted.source_type, + source_id=accepted.source_id, + time_kind=accepted.assertion.time_kind.value, + range_axis=valid_during.axis.value, + lower_value=valid_during.lower, + upper_value=valid_during.upper, + lower_inclusive=valid_during.lower_inclusive, + upper_inclusive=valid_during.upper_inclusive, + is_empty=valid_during.is_empty, + extractor=accepted.assertion.extractor, + source_text=accepted.assertion.source_text, + assertion_metadata=accepted.assertion.metadata, + ) + + +class MemoryTimeIndexRepository(Repository[MemoryTimeIndex]): + """Repository for the temporal projection of accepted note content.""" + + project_id: int + + def __init__(self, project_id: int): + """Initialize with project_id filter. + + Args: + project_id: Project ID to filter all operations by + """ + super().__init__(MemoryTimeIndex, project_id=project_id) + + async def find_by_entity( + self, session: AsyncSession, entity_id: int + ) -> Sequence[MemoryTimeIndex]: + """Find every temporal assertion projected from one entity.""" + query = ( + self.select() + .filter(MemoryTimeIndex.entity_id == entity_id) + .order_by(MemoryTimeIndex.source_id, MemoryTimeIndex.id) + ) + result = await self.execute_query(session, query) + return result.scalars().all() + + async def find_for_sources( + self, + session: AsyncSession, + sources: Iterable[tuple[str, int]], + ) -> Sequence[MemoryTimeIndex]: + """Find every assertion carried by the given ``(source_type, source_id)`` rows. + + Used to explain search hits, so it batches: search returns a page of rows and + this loads their assertions in one pass per source type rather than one query + per hit. Ids are chunked because SQLite caps bound parameters per statement. + """ + ids_by_type: defaultdict[str, list[int]] = defaultdict(list) + for source_type, source_id in sources: + ids_by_type[source_type].append(source_id) + if not ids_by_type: + return [] + + rows: list[MemoryTimeIndex] = [] + for source_type, source_ids in ids_by_type.items(): + for start in range(0, len(source_ids), SELECT_BY_IDS_CHUNK_SIZE): + chunk = source_ids[start : start + SELECT_BY_IDS_CHUNK_SIZE] + query = ( + self.select() + .filter(MemoryTimeIndex.source_type == source_type) + .filter(MemoryTimeIndex.source_id.in_(chunk)) + .order_by(MemoryTimeIndex.source_id, MemoryTimeIndex.id) + ) + result = await self.execute_query(session, query) + rows.extend(result.scalars().all()) + return rows + + async def replace_assertions_for_generation( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + assertions: Sequence[AcceptedTemporalAssertion], + ) -> TemporalGenerationWriteResult: + """Replace temporal rows only while the accepted content generation is current.""" + # This helper is a shared note_content fence despite its historical relation name. + current_generation = await session.scalar( + current_relation_generation_statement( + project_id=self.project_id, + entity_id=entity_id, + generation=generation, + ) + ) + # Trigger: a newer accepted note generation won before this transaction acquired the row. + # Why: replacing here would publish valid time the current markdown no longer asserts. + # Outcome: leave every existing row untouched and let the current writer publish. + if current_generation is None: + return TemporalGenerationWriteResult(generation_is_current=False) + + await self.delete_by_fields(session, entity_id=entity_id) + rows = [ + _projection_row(accepted, project_id=self.project_id, entity_id=entity_id) + for accepted in assertions + ] + await self.add_all_no_return(session, rows) + return TemporalGenerationWriteResult(generation_is_current=True) diff --git a/src/basic_memory/repository/note_type_filters.py b/src/basic_memory/repository/note_type_filters.py new file mode 100644 index 000000000..9da86df09 --- /dev/null +++ b/src/basic_memory/repository/note_type_filters.py @@ -0,0 +1,75 @@ +"""SQL for the note-type search predicate. + +A note type is a property of the *note*, not of the individual rows projected from it. +One markdown file becomes several search rows -- the entity itself, one per observation, +one per outgoing relation -- and only the entity row carries `metadata.note_type`, because +that is where the frontmatter lives. An observation row carries `metadata.tags`; a relation +row carries no metadata at all. + +Reading the type off each row therefore answers "is this row an entity of type X?" when the +question asked was "does this row belong to a note of type X?". Those coincide for entity +rows and for nothing else, which is invisible until a filter selects non-entity rows -- as a +valid-time filter does, since authored time lives on observations (SPEC-82). The conjunction +of the two was unsatisfiable: every row admitted by the temporal predicate was excluded by +the note-type one. + +Resolving through the owning note fixes that at the source. Every search row already carries +`entity_id`, and an entity row's own `id` equals it, so one membership test covers all three +row kinds without special-casing any of them and without copying the type onto rows that +would then have to be kept in step with the note's frontmatter. + +The predicate is a *non-correlated* subquery for the reason `temporal_filters` documents at +length: SQLite's `search_index` is an FTS5 virtual table, and a correlated `EXISTS` beside a +`MATCH` makes SQLite refuse the statement outright. A non-correlated `IN` is evaluated once, +independently, and composes with every FTS shape in this repository while leaving bm25 +ranking intact. + +One builder serves both dialects. Only the JSON accessor differs, so that is the single +thing a backend supplies -- the rule itself lives here rather than being written out once +per backend and drifting. +""" + +from __future__ import annotations + +from typing import Any, Sequence + +from basic_memory.schemas.search import SearchItemType + +SEARCH_TABLE = "search_index" + +# The alias the owning note's row carries inside the subquery. +_OWNER = "note_type_owner" + +# Each dialect's expression for the owning note's frontmatter type. +SQLITE_NOTE_TYPE_VALUE = f"json_extract({_OWNER}.metadata, '$.note_type')" +POSTGRES_NOTE_TYPE_VALUE = f"{_OWNER}.metadata->>'note_type'" + + +def build_note_type_predicate( + note_types: Sequence[str], + params: dict[str, Any], + *, + note_type_value: str, +) -> str: + """Build the WHERE-clause fragment restricting rows to notes of the given types. + + The stored type keeps the frontmatter's own casing (`Chapter`), while the filter is + documented case-insensitive, so both sides are folded to lowercase. + + Binds are added to `params` in place, following the convention the surrounding FTS + query builders already use. `project_id` is bound by the caller for the whole query. + """ + placeholders = [] + for index, note_type in enumerate(note_types): + name = f"note_type_{index}" + params[name] = note_type.lower() + placeholders.append(f":{name}") + + return ( + f"{SEARCH_TABLE}.entity_id IN (\n" + f" SELECT {_OWNER}.id\n" + f" FROM {SEARCH_TABLE} AS {_OWNER}\n" + f" WHERE {_OWNER}.type = '{SearchItemType.ENTITY.value}'\n" + f" AND {_OWNER}.project_id = :project_id\n" + f" AND LOWER({note_type_value}) IN ({', '.join(placeholders)}))" + ) diff --git a/src/basic_memory/repository/observation_repository.py b/src/basic_memory/repository/observation_repository.py index 57e18c86a..7424ee353 100644 --- a/src/basic_memory/repository/observation_repository.py +++ b/src/basic_memory/repository/observation_repository.py @@ -9,8 +9,10 @@ from sqlalchemy.orm.interfaces import LoaderOption from basic_memory.models import Observation +from basic_memory.models.knowledge import observation_permalink_tail from basic_memory.repository.relation_repository import current_relation_generation_statement from basic_memory.repository.repository import Repository +from basic_memory.temporal import TemporalAssertion @dataclass(frozen=True, slots=True) @@ -20,19 +22,30 @@ class AcceptedObservationWrite: Mirrors the markdown ``Observation`` fields so the accepted-write path can persist the graph without constructing ORM rows in the storage-neutral runner (issue #1076). + + ``temporal`` rides along rather than becoming observation columns: authored + valid time is its own projection keyed on the row this write mints, and an + observation may carry several assertions (SPEC-82). """ content: str category: str | None context: str | None tags: list[str] | None + temporal: tuple[TemporalAssertion, ...] = () @dataclass(frozen=True, slots=True) class ObservationGenerationWriteResult: - """Whether a guarded observation replacement still owned its source generation.""" + """Whether a guarded observation replacement still owned its source generation. + + ``observation_ids`` are the freshly minted row ids in document order, aligned + with the sequence that was written. The temporal projection addresses those + rows, and they only exist once the insert has flushed. + """ generation_is_current: bool + observation_ids: tuple[int, ...] = () class ObservationRepository(Repository[Observation]): @@ -130,16 +143,38 @@ async def replace_observations_for_generation( return ObservationGenerationWriteResult(generation_is_current=False) await self.delete_by_fields(session, entity_id=entity_id) - rows = [ - Observation( - project_id=self.project_id, - entity_id=entity_id, - content=obs.content, - category=obs.category, - context=obs.context, - tags=obs.tags, + # A note may say the same thing twice and mean two different things -- most + # sharply when a temporal qualifier or a (context) is what separates them, since + # both are peeled off before the content reaches this row. The permalink is built + # from what survives that peel, so those twins would address one row, and the + # permalink-keyed search index would keep only the first (SPEC-82). + # + # This is the one place that sees a note's whole observation set in document + # order, so it is where the ordinal that separates them can be counted at all. + # `observation_permalink_tail` is shared with `Observation.permalink` so the + # count is taken over exactly the identity the address is built from. + duplicates_seen: dict[str, int] = {} + rows = [] + for obs in observations: + identity = observation_permalink_tail(obs.category, obs.content) + duplicate_index = duplicates_seen.get(identity, 0) + duplicates_seen[identity] = duplicate_index + 1 + rows.append( + Observation( + project_id=self.project_id, + entity_id=entity_id, + content=obs.content, + category=obs.category, + context=obs.context, + tags=obs.tags, + duplicate_index=duplicate_index, + ) ) - for obs in observations - ] await self.add_all_no_return(session, rows) - return ObservationGenerationWriteResult(generation_is_current=True) + # add_all_no_return flushes, so every row now carries its database id. + # Reading them here, inside the same transaction, is what lets the temporal + # projection address these exact rows. + return ObservationGenerationWriteResult( + generation_is_current=True, + observation_ids=tuple(row.id for row in rows), + ) diff --git a/src/basic_memory/repository/postgres_search_repository.py b/src/basic_memory/repository/postgres_search_repository.py index eb8a3f4e3..b187b4053 100644 --- a/src/basic_memory/repository/postgres_search_repository.py +++ b/src/basic_memory/repository/postgres_search_repository.py @@ -35,6 +35,11 @@ build_fts_page_stage, ) from basic_memory.repository.metadata_filters import parse_metadata_filters +from basic_memory.repository.note_type_filters import ( + POSTGRES_NOTE_TYPE_VALUE, + build_note_type_predicate, +) +from basic_memory.repository.temporal_filters import build_temporal_predicate from basic_memory.repository.semantic_errors import SemanticDependenciesMissingError from basic_memory.repository.semantic_vector_index import SemanticVectorIndex from basic_memory.repository.semantic_vector_sync import ( @@ -48,6 +53,7 @@ from basic_memory.repository.pgvector_index import PgVectorIndex from basic_memory.repository.postgres_fts_chunks import split_postgres_fts_chunks from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode +from basic_memory.temporal import TemporalFilter _TSQUERY_OPERAND_PATTERN = re.compile(r"'(?:''|[^'])*'(?::\*)?|[^\s&|!()]+") @@ -972,6 +978,7 @@ async def _build_fts_query_parts( categories: Optional[List[str]] = None, metadata_filters: Optional[dict[str, Any]] = None, file_path_prefix: Optional[str] = None, + temporal: Optional[TemporalFilter] = None, allow_relaxed: bool = False, ) -> tuple[str, str, dict[str, Any], str, str]: """Build Postgres FTS FROM/WHERE params shared by search and count.""" @@ -1166,19 +1173,18 @@ async def _build_fts_query_parts( # Handle note type filter (frontmatter type field, parameterized). # Trigger: caller passed `note_types` to scope by the frontmatter `type` field. - # Why: the stored note_type preserves the frontmatter casing (e.g. `Chapter`), - # but the filter is documented case-insensitive. JSONB `@>` containment is - # exact-match, so capitalized types were unfindable. - # Outcome: compare LOWER(metadata->>'note_type') against lowercased filter - # values so `note_types=["Chapter"]` matches a stored `Chapter`. + # Why: the type belongs to the note, but only its entity row carries the + # frontmatter; observation and relation rows do not. Reading it off each row + # silently excluded every non-entity row, which made `note_types` combined + # with a valid-time filter unsatisfiable. + # Outcome: resolved through the owning note in one shared builder, so both + # backends ask the same question and observation rows of a matching note + # are admitted. if note_types: - type_placeholders = [] - for idx, note_type in enumerate(note_types): - param_name = f"note_type_{idx}" - params[param_name] = note_type.lower() - type_placeholders.append(f":{param_name}") conditions.append( - f"LOWER(search_index.metadata->>'note_type') IN ({', '.join(type_placeholders)})" + build_note_type_predicate( + note_types, params, note_type_value=POSTGRES_NOTE_TYPE_VALUE + ) ) # Handle date filter @@ -1189,6 +1195,18 @@ async def _build_fts_query_parts( # order by most recent first order_by_clause = ", search_index.updated_at DESC" + # Handle authored valid time (SPEC-82). + # Trigger: caller asked when a statement was true of the world. + # Why: `after_date` above filters `updated_at`, which records when the note was + # last edited. That is bookkeeping, never a semantic claim; a decision + # effective through July says nothing about when its file was touched. + # Outcome: an independent predicate over the temporal projection, textually + # identical to the SQLite one because canonical bounds compare + # lexicographically on both backends. Undated sources carry no row and + # are therefore excluded whenever a valid-time filter is present. + if temporal is not None: + conditions.append(build_temporal_predicate(temporal, params)) + # Handle structured metadata filters (frontmatter) # Uses jsonb_extract_path_text() / jsonb_extract_path() with parameterized # path parts instead of #>> / #> with interpolated paths. @@ -1368,6 +1386,7 @@ async def search( categories: Optional[List[str]] = None, metadata_filters: Optional[dict[str, Any]] = None, file_path_prefix: Optional[str] = None, + temporal: Optional[TemporalFilter] = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: Optional[float] = None, limit: int = 10, @@ -1390,6 +1409,7 @@ async def search( categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, retrieval_mode=retrieval_mode, min_similarity=min_similarity, limit=limit, @@ -1417,6 +1437,7 @@ async def search( categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, allow_relaxed=allow_relaxed, ) @@ -1564,6 +1585,7 @@ async def count( categories: Optional[List[str]] = None, metadata_filters: Optional[dict[str, Any]] = None, file_path_prefix: Optional[str] = None, + temporal: Optional[TemporalFilter] = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: Optional[float] = None, allow_relaxed: bool = False, @@ -1581,6 +1603,7 @@ async def count( categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, retrieval_mode=retrieval_mode, min_similarity=min_similarity, ) @@ -1602,6 +1625,7 @@ async def count( categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, allow_relaxed=allow_relaxed, ) sql = f"SELECT COUNT(*) FROM {from_clause} WHERE {where_clause}" diff --git a/src/basic_memory/repository/search_repository.py b/src/basic_memory/repository/search_repository.py index ace8e329e..4d7685128 100644 --- a/src/basic_memory/repository/search_repository.py +++ b/src/basic_memory/repository/search_repository.py @@ -27,6 +27,7 @@ from basic_memory.runtime.vector_sync import VectorSyncBatchResult from basic_memory.repository.sqlite_search_repository import SQLiteSearchRepository from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode +from basic_memory.temporal import TemporalFilter class SearchRepository(Protocol): @@ -80,6 +81,7 @@ async def search( categories: Optional[List[str]] = None, metadata_filters: Optional[dict[str, Any]] = None, file_path_prefix: Optional[str] = None, + temporal: Optional[TemporalFilter] = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: Optional[float] = None, limit: int = 10, @@ -104,6 +106,7 @@ async def count( categories: Optional[List[str]] = None, metadata_filters: Optional[dict[str, Any]] = None, file_path_prefix: Optional[str] = None, + temporal: Optional[TemporalFilter] = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: Optional[float] = None, allow_relaxed: bool = False, diff --git a/src/basic_memory/repository/search_repository_base.py b/src/basic_memory/repository/search_repository_base.py index daa561778..acc212f40 100644 --- a/src/basic_memory/repository/search_repository_base.py +++ b/src/basic_memory/repository/search_repository_base.py @@ -83,6 +83,7 @@ SearchRetrievalMode, normalize_file_path_prefix, ) +from basic_memory.temporal import TemporalFilter from basic_memory.utils import ensure_timezone_aware # --- Semantic search constants --- @@ -416,6 +417,7 @@ async def search( categories: Optional[List[str]] = None, metadata_filters: Optional[Dict[str, Any]] = None, file_path_prefix: Optional[str] = None, + temporal: Optional[TemporalFilter] = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: Optional[float] = None, limit: int = 10, @@ -437,6 +439,9 @@ async def search( categories: Filter observations by exact category (e.g. "requirement") metadata_filters: Structured frontmatter metadata filters file_path_prefix: Directory subtree scope, matched against file_path + temporal: Authored valid-time filter. Unlike after_date, which reads the + note's edit bookkeeping, this reads the time an observation claims to + be true of the world. Sources without such a claim are excluded. limit: Maximum results to return offset: Number of results to skip @@ -461,6 +466,7 @@ async def count( categories: Optional[List[str]] = None, metadata_filters: Optional[Dict[str, Any]] = None, file_path_prefix: Optional[str] = None, + temporal: Optional[TemporalFilter] = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: Optional[float] = None, allow_relaxed: bool = False, @@ -2014,6 +2020,7 @@ async def _dispatch_retrieval_mode( categories: Optional[List[str]], metadata_filters: Optional[dict[str, Any]], file_path_prefix: Optional[str], + temporal: Optional[TemporalFilter], retrieval_mode: SearchRetrievalMode, min_similarity: Optional[float] = None, limit: int, @@ -2050,6 +2057,7 @@ async def _dispatch_retrieval_mode( categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, min_similarity=min_similarity, limit=limit, offset=offset, @@ -2072,6 +2080,7 @@ async def _dispatch_retrieval_mode( categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, min_similarity=min_similarity, limit=limit, offset=offset, @@ -2251,6 +2260,7 @@ async def _search_vector_only( categories: Optional[List[str]], metadata_filters: Optional[dict[str, Any]], file_path_prefix: Optional[str], + temporal: Optional[TemporalFilter], min_similarity: Optional[float] = None, limit: int, offset: int, @@ -2439,6 +2449,7 @@ def _log_vector_summary() -> None: categories, metadata_filters, file_path_prefix, + temporal, ] ) @@ -2454,6 +2465,7 @@ def _log_vector_summary() -> None: categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, retrieval_mode=SearchRetrievalMode.FTS, limit=VECTOR_FILTER_SCAN_LIMIT, offset=0, @@ -2518,6 +2530,7 @@ def _log_vector_summary() -> None: categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, min_similarity=min_similarity, limit=stable_candidate_limit, offset=0, @@ -2591,6 +2604,7 @@ async def _search_hybrid( categories: Optional[List[str]], metadata_filters: Optional[dict[str, Any]], file_path_prefix: Optional[str], + temporal: Optional[TemporalFilter], min_similarity: Optional[float] = None, limit: int, offset: int, @@ -2630,6 +2644,7 @@ async def _search_hybrid( categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, retrieval_mode=SearchRetrievalMode.FTS, limit=candidate_limit, offset=0, @@ -2649,6 +2664,7 @@ async def _search_hybrid( categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, min_similarity=min_similarity, limit=candidate_limit, offset=0, @@ -2800,6 +2816,7 @@ def _materialize(entry: tuple[SearchIndexKey, float]) -> SearchIndexRow: categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, min_similarity=min_similarity, limit=stable_candidate_limit, offset=0, diff --git a/src/basic_memory/repository/sqlite_search_repository.py b/src/basic_memory/repository/sqlite_search_repository.py index 8f5983066..d48a737d7 100644 --- a/src/basic_memory/repository/sqlite_search_repository.py +++ b/src/basic_memory/repository/sqlite_search_repository.py @@ -40,12 +40,18 @@ build_fts_page_stage, ) from basic_memory.repository.metadata_filters import parse_metadata_filters, build_sqlite_json_path +from basic_memory.repository.note_type_filters import ( + SQLITE_NOTE_TYPE_VALUE, + build_note_type_predicate, +) +from basic_memory.repository.temporal_filters import build_temporal_predicate from basic_memory.repository.semantic_errors import SemanticDependenciesMissingError from basic_memory.repository.semantic_vector_index import SemanticVectorIndex from basic_memory.repository.semantic_vector_sync import StagedVectorDeletion from basic_memory.repository.semantic_vector_index_factory import build_vector_index_scope from basic_memory.repository.sqlite_vec_index import SQLiteVecIndex from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode +from basic_memory.temporal import TemporalFilter SQLITE_WORD_COLUMNS = "{title content_stems content_snippet}" @@ -788,6 +794,7 @@ async def _build_fts_query_parts( categories: Optional[List[str]] = None, metadata_filters: Optional[dict[str, Any]] = None, file_path_prefix: Optional[str] = None, + temporal: Optional[TemporalFilter] = None, ) -> tuple[str, str, dict[str, Any], str, str]: """Build SQLite FTS FROM/WHERE params shared by search and count.""" conditions = [] @@ -906,20 +913,18 @@ async def _build_fts_query_parts( # Handle note type filter (frontmatter type field, parameterized). # Trigger: caller passed `note_types` to scope by the frontmatter `type` field. - # Why: the stored note_type preserves the frontmatter casing (e.g. `Chapter`), - # but the filter is documented case-insensitive; comparing raw values - # would miss capitalized types. - # Outcome: fold both sides to lowercase so `note_types=["Chapter"]` matches a - # stored `Chapter`, `chapter`, etc. + # Why: the type belongs to the note, but only its entity row carries the + # frontmatter; observation and relation rows do not. Reading it off each row + # silently excluded every non-entity row, which made `note_types` combined + # with a valid-time filter unsatisfiable. + # Outcome: resolved through the owning note in one shared builder, so both + # backends ask the same question and observation rows of a matching note + # are admitted. if note_types: - type_placeholders = [] - for idx, t in enumerate(note_types): - param_name = f"note_type_{idx}" - params[param_name] = t.lower() - type_placeholders.append(f":{param_name}") conditions.append( - "LOWER(json_extract(search_index.metadata, '$.note_type')) " - f"IN ({', '.join(type_placeholders)})" + build_note_type_predicate( + note_types, params, note_type_value=SQLITE_NOTE_TYPE_VALUE + ) ) # Handle date filter using datetime() for proper comparison @@ -931,6 +936,18 @@ async def _build_fts_query_parts( # order by most recent first order_by_clause = ", search_index.updated_at DESC" + # Handle authored valid time (SPEC-82). + # Trigger: caller asked when a statement was true of the world. + # Why: `after_date` above filters `updated_at`, which records when the note was + # last edited. That is bookkeeping, never a semantic claim; a decision + # effective through July says nothing about when its file was touched. + # Outcome: an independent predicate over the temporal projection. It matches + # only sources carrying a structured qualifier, so undated sources are + # excluded whenever a valid-time filter is present, and no ordering + # changes -- relevance still decides the ranking. + if temporal is not None: + conditions.append(build_temporal_predicate(temporal, params)) + # Handle structured metadata filters (frontmatter) if metadata_filters: parsed_filters = parse_metadata_filters(metadata_filters) @@ -1085,6 +1102,7 @@ async def search( categories: Optional[List[str]] = None, metadata_filters: Optional[dict[str, Any]] = None, file_path_prefix: Optional[str] = None, + temporal: Optional[TemporalFilter] = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: Optional[float] = None, limit: int = 10, @@ -1113,6 +1131,7 @@ async def search( categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, retrieval_mode=retrieval_mode, min_similarity=min_similarity, limit=limit, @@ -1140,6 +1159,7 @@ async def search( categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, ) # set limit on search query @@ -1272,6 +1292,7 @@ async def count( categories: Optional[List[str]] = None, metadata_filters: Optional[dict[str, Any]] = None, file_path_prefix: Optional[str] = None, + temporal: Optional[TemporalFilter] = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: Optional[float] = None, allow_relaxed: bool = False, @@ -1289,6 +1310,7 @@ async def count( categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, retrieval_mode=retrieval_mode, min_similarity=min_similarity, ) @@ -1310,6 +1332,7 @@ async def count( categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, ) sql = f"SELECT COUNT(*) FROM {from_clause} WHERE {where_clause}" logger.trace(f"Count {sql} params: {params}") diff --git a/src/basic_memory/repository/temporal_filters.py b/src/basic_memory/repository/temporal_filters.py new file mode 100644 index 000000000..81c3c666c --- /dev/null +++ b/src/basic_memory/repository/temporal_filters.py @@ -0,0 +1,145 @@ +"""SQL for the valid-time search predicate (SPEC-82). + +One builder serves both dialects. That is not a coincidence to be maintained by +discipline -- it falls out of two decisions made upstream: + +* `basic_memory.temporal` canonicalizes every bound to a fixed-width lexical form, + so `<`, `>`, and `=` on plain text columns *are* chronological comparisons and no + typed date bind is needed on either side. +* Inclusivity on the query side is known while the SQL is being built, and + inclusivity on the stored side is a boolean column, so both fold into the SQL text. + The only bound parameters are the two bound values, the kind, and the axis -- each + compared directly against a column, so PostgreSQL always infers their type and + asyncpg never sees a bare untyped parameter. + +Comparing endpoint *values* like this is only equivalent to comparing the sets of times +they delimit because `TemporalRange` has already canonicalized every date range to the +half-open `[lower,upper)` form. Calendar dates are discrete, so two date ranges can hold +each other's raw endpoints while sharing no actual day; the canonical form is what rules +that out. The inclusivity branches below therefore only ever fire on the instant axis in +practice, and they stay because instants are continuous and keep the authored form. + +The predicate is a *non-correlated* subquery, and that shape is load-bearing rather +than stylistic. SQLite's default word search emits an OR of per-column `MATCH` +predicates; adding a correlated `EXISTS` to that WHERE clause makes SQLite refuse the +statement outright ("unable to use function MATCH in the requested context"). A +non-correlated `IN` is evaluated independently and composes with every FTS shape in +this repository -- the OR-of-columns form, the table-level `MATCH` used for script +queries, the bm25-preserving derived table, and the rowid rewrite -- while leaving +bm25 ranking intact. It also needs no change to any `from_clause`. +""" + +from __future__ import annotations + +from typing import Any + +from basic_memory.temporal import TemporalFilter, TemporalRange + +TEMPORAL_INDEX_TABLE = "memory_time_index" + +# No stored assertion can match, and no subquery needs to run to prove it. +_MATCHES_NOTHING = "1 = 0" + + +def _not_source_ends_before_window(window: TemporalRange) -> str | None: + """Reject stored ranges that finish before the queried window begins. + + Returns None when the window is unbounded below, because then nothing can end + before it starts and the whole conjunct is vacuous. + """ + if window.lower is None: + return None + clauses = [ + # An unbounded stored upper end never terminates, so it can never be "before". + f"{TEMPORAL_INDEX_TABLE}.upper_value IS NULL", + f"{TEMPORAL_INDEX_TABLE}.upper_value > :tq_lower", + ] + if window.lower_inclusive: + # The window owns its lower endpoint, so a stored range that closes on that + # same endpoint still shares it. + clauses.append( + f"({TEMPORAL_INDEX_TABLE}.upper_value = :tq_lower " + f"AND {TEMPORAL_INDEX_TABLE}.upper_inclusive)" + ) + return f"({' OR '.join(clauses)})" + + +def _not_window_ends_before_source(window: TemporalRange) -> str | None: + """Reject stored ranges that begin after the queried window ends. + + The mirror image of `_not_source_ends_before_window`; None when the window is + unbounded above. + """ + if window.upper is None: + return None + clauses = [ + f"{TEMPORAL_INDEX_TABLE}.lower_value IS NULL", + f"{TEMPORAL_INDEX_TABLE}.lower_value < :tq_upper", + ] + if window.upper_inclusive: + clauses.append( + f"({TEMPORAL_INDEX_TABLE}.lower_value = :tq_upper " + f"AND {TEMPORAL_INDEX_TABLE}.lower_inclusive)" + ) + return f"({' OR '.join(clauses)})" + + +def build_temporal_predicate(temporal: TemporalFilter, params: dict[str, Any]) -> str: + """Build the WHERE-clause fragment restricting search rows by authored valid time. + + Two intervals overlap exactly when neither lies entirely before the other, which + is what the two helpers above assert. Containment of a single date or instant is + the same question asked of the degenerate closed range `[p,p]`, so `valid_at` and + `valid_overlaps` share this one implementation and cannot drift apart. + + The result matches only sources carrying a structured assertion: a note without a + qualifier contributes no row here and is therefore excluded, which is the + documented default for a valid-time query. + + Binds are added to `params` in place, following the convention already used by the + surrounding FTS query builders. + """ + window = temporal.window + if window is not None and window.is_empty: + # PostgreSQL: nothing overlaps the empty range, not even itself. Emitting a + # false constant is both correct and cheaper than running the subquery. + return _MATCHES_NOTHING + + conditions = [f"{TEMPORAL_INDEX_TABLE}.project_id = :project_id"] + + if temporal.kind is not None: + params["tq_kind"] = temporal.kind.value + conditions.append(f"{TEMPORAL_INDEX_TABLE}.time_kind = :tq_kind") + + if window is not None: + # Trigger: the caller asked about a specific date or a specific instant. + # Why: calendar dates and instants are different axes; converting between + # them would invent a timezone or a time of day the author never wrote. + # Outcome: a date query can never match an instant range, or the reverse. + params["tq_axis"] = window.axis.value + conditions.append(f"{TEMPORAL_INDEX_TABLE}.range_axis = :tq_axis") + # The empty stored range contains no points, so it overlaps nothing. + conditions.append(f"NOT {TEMPORAL_INDEX_TABLE}.is_empty") + + if window.lower is not None: + params["tq_lower"] = window.lower + if window.upper is not None: + params["tq_upper"] = window.upper + conditions.extend( + clause + for clause in ( + _not_source_ends_before_window(window), + _not_window_ends_before_source(window), + ) + if clause is not None + ) + + where_clause = "\n AND ".join(conditions) + # (type, id) is the search row's own identity and the address this projection + # stores, so the pair joins the two without a correlated reference. + return ( + "(search_index.type, search_index.id) IN (\n" + f" SELECT {TEMPORAL_INDEX_TABLE}.source_type, {TEMPORAL_INDEX_TABLE}.source_id\n" + f" FROM {TEMPORAL_INDEX_TABLE}\n" + f" WHERE {where_clause})" + ) diff --git a/src/basic_memory/schemas/search.py b/src/basic_memory/schemas/search.py index 749d93243..4ef4120a1 100644 --- a/src/basic_memory/schemas/search.py +++ b/src/basic_memory/schemas/search.py @@ -9,7 +9,7 @@ from typing import Optional, List, Union, Any from datetime import datetime from enum import Enum -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field, field_validator, model_validator from basic_memory.schemas.base import Permalink, normalize_note_type @@ -77,6 +77,12 @@ class SearchQuery(BaseModel): - file_path_prefix: Limit to one directory subtree of the project - tags: Convenience frontmatter tag filter - status: Convenience frontmatter status filter + - valid_at / valid_overlaps / time_kind: Authored valid-time filters (SPEC-82) + + Valid time is what a note *says about the world*, written as a qualifier on an + observation (``- [decision] @effective[2026-06-10,2026-07-27) ...``). It is a + different axis from ``after_date``, which filters on when a row was last indexed + and is deliberately left untouched by these fields. Boolean search examples: - "python AND flask" - Find items with both terms @@ -106,6 +112,22 @@ class SearchQuery(BaseModel): retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS min_similarity: Optional[float] = None # Per-query override for semantic_min_similarity + # Authored valid-time filters. Kept as strings at the boundary so HTTP clients and + # MCP callers can pass one flat value; the service parses them into the portable + # domain values and rejects anything malformed with a visible diagnostic. + valid_at: Optional[str] = None # Date or RFC 3339 instant the range must contain + valid_overlaps: Optional[str] = None # Range literal, e.g. "[2026-06-10,2026-07-27)" + time_kind: Optional[str] = None # effective | valid | occurred | due | mentioned + + @model_validator(mode="after") + def validate_temporal_filter(self) -> "SearchQuery": + """Refuse a query that asks two different valid-time questions at once.""" + if self.valid_at is not None and self.valid_overlaps is not None: + raise ValueError( + "Use either valid_at (containment) or valid_overlaps (overlap), not both." + ) + return self + @field_validator("after_date") @classmethod def validate_date(cls, v: Optional[Union[datetime, str]]) -> Optional[str]: @@ -133,6 +155,15 @@ def normalize_scope(cls, value: Optional[str]) -> Optional[str]: """ return normalize_file_path_prefix(value) + def has_temporal_filter(self) -> bool: + """Whether this query asks a valid-time question at all. + + A kind on its own is a legal filter: it asks for sources carrying any + assertion of that kind. Callers use this to decide whether valid time was + requested without parsing the values, which is why it never raises. + """ + return bool(self.valid_at or self.valid_overlaps or self.time_kind) + def no_criteria(self) -> bool: text_is_empty = self.text is None or (isinstance(self.text, str) and not self.text.strip()) metadata_is_empty = not self.metadata_filters @@ -155,6 +186,7 @@ def no_criteria(self) -> bool: and self.file_path_prefix is None and tags_is_empty and status_is_empty + and not self.has_temporal_filter() ) def has_boolean_operators(self) -> bool: @@ -169,6 +201,41 @@ def has_boolean_operators(self) -> bool: return any(pattern in text for pattern in boolean_patterns) +class TemporalRangeValue(BaseModel): + """One authored interval, as a caller sees it. + + This is the single logical `valid_during` value the API and MCP boundary expose. + How the projection stores it -- which table, which columns, which indexes -- is + deliberately absent: `literal` is the canonical PostgreSQL range literal and the + decomposed bounds are the same interval, spelled out so a caller can compare + endpoints without parsing. + + Canonical means canonical: a *date* range always reads half-open, whatever brackets + the author typed, because calendar dates are discrete. `source_text` on the + enclosing `TemporalResultMetadata` is where the author's own spelling survives. + """ + + axis: str # "date" (calendar dates) or "instant" (UTC timestamps) + literal: str # e.g. "[2026-06-10,2026-07-28)", "(,2026-07-27)", "empty" + lower: Optional[str] = None # None means unbounded on that side + upper: Optional[str] = None + lower_inclusive: bool = False + upper_inclusive: bool = False + is_empty: bool = False + + +class TemporalResultMetadata(BaseModel): + """One authored valid-time assertion carried by a search result. + + Present so an agent can say *why* a source matched a valid-time query -- which + kind of time it asserts, over what interval, and in the author's own words. + """ + + kind: str # effective | valid | occurred | due | mentioned + valid_during: TemporalRangeValue + source_text: str # the qualifier exactly as authored, e.g. "@effective[2026-06-10,)" + + class SearchResult(BaseModel): """Search result with score and metadata.""" @@ -199,6 +266,11 @@ class SearchResult(BaseModel): to_entity: Optional[Permalink] = None # For relations relation_type: Optional[str] = None # For relations + # Authored valid-time assertions carried by this row. Collection-shaped from day + # one: the MVP parser reads one qualifier per observation, but multiple assertions + # of multiple kinds must not be a schema break later. + temporal: Optional[List[TemporalResultMetadata]] = None + class SearchResponse(BaseModel): """Wrapper for search results.""" @@ -215,3 +287,17 @@ class SearchResponse(BaseModel): description="Whether total is an exact count that clients can use for pagination", ) has_more: bool = False + # Version-skew guard. SearchQuery ignores unknown fields, so a client that sends a + # valid-time filter to a server predating SPEC-82 would receive unfiltered results + # that look filtered -- silently including the undated sources the filter excludes. + # + # Three states, all meaningful: True (asked and executed), None (never asked, so + # nothing to confirm), and -- only from a server that does not know this field -- + # missing, which parses as None while True was expected. Staying None rather than + # False when no filter was asked keeps every ordinary search payload byte-identical + # to what it was before valid time existed. + temporal_applied: Optional[bool] = Field( + default=None, + description="True when the server executed a requested valid-time filter; " + "absent when the request carried none", + ) diff --git a/src/basic_memory/services/entity_service.py b/src/basic_memory/services/entity_service.py index 715c52cfc..d142b0cd6 100644 --- a/src/basic_memory/services/entity_service.py +++ b/src/basic_memory/services/entity_service.py @@ -30,6 +30,7 @@ from basic_memory.repository import ObservationRepository, RelationRepository from basic_memory.repository.entity_repository import EntityRepository from basic_memory.repository.note_content_repository import NoteContentRepository +from basic_memory.repository.memory_time_index_repository import MemoryTimeIndexRepository from basic_memory.repository.note_section_repository import NoteSectionRepository from basic_memory.read_cache import ReadCache, invalidate_cache from basic_memory.runtime.note_move import normalize_note_move_destination_path @@ -233,6 +234,7 @@ async def _publish_markdown_graph( category=observation.category, context=observation.context, tags=observation.tags, + temporal=tuple(observation.temporal), ) for observation in markdown.observations ) @@ -252,6 +254,7 @@ async def _publish_markdown_graph( relation_repository=self.relation_repository, observation_repository=self.observation_repository, section_repository=NoteSectionRepository(project_id=self.repository.project_id), + temporal_repository=MemoryTimeIndexRepository(project_id=self.repository.project_id), session_maker=self.session_maker, ) published = await publisher.publish( diff --git a/src/basic_memory/services/note_content_writes.py b/src/basic_memory/services/note_content_writes.py index 1143ba745..4da33ab45 100644 --- a/src/basic_memory/services/note_content_writes.py +++ b/src/basic_memory/services/note_content_writes.py @@ -175,10 +175,14 @@ async def _publish_relation_generation( section_repository = self.mutation_dependencies.write_repositories.section_repository( publication.project_id ) + temporal_repository = self.mutation_dependencies.write_repositories.temporal_repository( + publication.project_id + ) publisher = RelationGenerationPublisher( relation_repository=repository, observation_repository=observation_repository, section_repository=section_repository, + temporal_repository=temporal_repository, session_maker=self.session_maker, ) await publisher.publish( diff --git a/src/basic_memory/services/note_preparation.py b/src/basic_memory/services/note_preparation.py index 23ed69e2a..e2508ed4d 100644 --- a/src/basic_memory/services/note_preparation.py +++ b/src/basic_memory/services/note_preparation.py @@ -95,6 +95,7 @@ def observations(self) -> list[AcceptedObservationWrite]: category=observation.category, context=observation.context, tags=observation.tags, + temporal=tuple(observation.temporal), ) for observation in self.entity_markdown.observations ] @@ -875,6 +876,7 @@ async def prepare_move_entity_content( category=observation.category, context=observation.context, tags=observation.tags, + temporal=tuple(observation.temporal), ) for observation in entity_markdown.observations ), diff --git a/src/basic_memory/services/search_service.py b/src/basic_memory/services/search_service.py index 588090365..18ee46f37 100644 --- a/src/basic_memory/services/search_service.py +++ b/src/basic_memory/services/search_service.py @@ -32,6 +32,10 @@ VectorSyncBatchResult, ) from basic_memory.services import FileService +from basic_memory.temporal import ( + TemporalFilter, + parse_temporal_filter, +) # Maximum size for content_stems field to stay under Postgres's 8KB index row limit. # We use 6000 characters to leave headroom for other indexed columns and overhead. @@ -52,6 +56,7 @@ class PreparedSearchQuery: after_date: datetime | None metadata_filters: dict[str, Any] | None file_path_prefix: str | None + temporal: TemporalFilter | None retrieval_mode: SearchRetrievalMode min_similarity: float | None @@ -84,6 +89,35 @@ def entity_embeddings_enabled(entity: Entity) -> bool: return True +def build_temporal_filter(query: SearchQuery) -> TemporalFilter | None: + """Read the query's flat valid-time fields as one portable filter value. + + The parsing itself lives in `temporal.parse_temporal_filter`, which every request + surface shares, so a caller that pre-validates the same three strings can never + disagree with what runs here. `TemporalQualifierError` is a `ValueError`, so callers + above map it to a 400. + """ + return parse_temporal_filter( + valid_at=query.valid_at, + valid_overlaps=query.valid_overlaps, + time_kind=query.time_kind, + ) + + +def _describe_temporal_criteria(temporal: TemporalFilter | None) -> str | None: + """Render the valid-time question that actually ran, for search traces.""" + if temporal is None: + return None + parts = [] + if temporal.kind is not None: + parts.append(f"kind={temporal.kind.value}") + if temporal.at is not None: + parts.append(f"valid_at={temporal.at.value}") + elif temporal.overlaps is not None: + parts.append(f"valid_overlaps={temporal.overlaps}") + return ",".join(parts) + + def describe_search_criteria(prepared: PreparedSearchQuery) -> str: """Render the criteria the repository actually executed. @@ -111,6 +145,7 @@ def quoted(value: str | None) -> str | None: "categories": list(prepared.categories) if prepared.categories else None, "metadata_filters": dict(prepared.metadata_filters) if prepared.metadata_filters else None, "file_path_prefix": quoted(prepared.file_path_prefix), + "temporal": _describe_temporal_criteria(prepared.temporal), } return " ".join(f"{name}={value}" for name, value in criteria.items() if value is not None) @@ -227,6 +262,7 @@ def prepare_query(self, query: SearchQuery) -> PreparedSearchQuery | None: after_date=after_date, metadata_filters=metadata_filters, file_path_prefix=query.file_path_prefix, + temporal=build_temporal_filter(query), retrieval_mode=query.retrieval_mode or SearchRetrievalMode.FTS, min_similarity=query.min_similarity, ) @@ -243,6 +279,7 @@ def prepare_query(self, query: SearchQuery) -> PreparedSearchQuery | None: or prepared.metadata_filters # Normalized by SearchQuery, so only a real subtree reaches here. or prepared.file_path_prefix + or prepared.temporal ) if not has_criteria: logger.debug("no criteria passed to query") @@ -258,6 +295,7 @@ def _prepared_has_filters(prepared: PreparedSearchQuery) -> bool: or prepared.categories or prepared.after_date or prepared.file_path_prefix + or prepared.temporal ) async def _include_legacy_note_type_spellings( @@ -312,6 +350,7 @@ async def _search_repository( after_date=prepared.after_date, metadata_filters=prepared.metadata_filters, file_path_prefix=prepared.file_path_prefix, + temporal=prepared.temporal, retrieval_mode=prepared.retrieval_mode, min_similarity=prepared.min_similarity, limit=limit, @@ -330,6 +369,7 @@ async def _search_repository( after_date=prepared.after_date, metadata_filters=prepared.metadata_filters, file_path_prefix=prepared.file_path_prefix, + temporal=prepared.temporal, retrieval_mode=prepared.retrieval_mode, min_similarity=prepared.min_similarity, limit=limit, @@ -357,6 +397,7 @@ async def _count_repository( after_date=prepared.after_date, metadata_filters=prepared.metadata_filters, file_path_prefix=prepared.file_path_prefix, + temporal=prepared.temporal, retrieval_mode=prepared.retrieval_mode, min_similarity=prepared.min_similarity, allow_relaxed=allow_relaxed, diff --git a/src/basic_memory/temporal.py b/src/basic_memory/temporal.py new file mode 100644 index 000000000..3b0c7e1a6 --- /dev/null +++ b/src/basic_memory/temporal.py @@ -0,0 +1,856 @@ +"""Portable temporal value types for authored valid time (SPEC-82). + +Basic Memory authors time as *semantic* data. A `[decision]` that was effective from +June 10 until the July 27 cutover is a statement about the world, not a record of when +the note was edited. This module owns the values that carry such a statement and the +lexical grammar for the range literals authors write. + +PostgreSQL's range conventions are the language contract: `[lower,upper)` with explicit +inclusivity per side, unbounded ends, and a distinguished empty range. That is a +vocabulary choice, not a storage requirement -- these values reduce to portable scalars +so SQLite and Postgres can share one logical model. Its *discrete* canonicalization is +part of the contract too: a date range is stored as `[lower,upper)`, for the reason +`TemporalRange` documents. The author's own spelling is not lost -- it is kept verbatim +on `TemporalAssertion.source_text`. + +Two canonical lexical forms carry every bound: + + date ``YYYY-MM-DD`` (10 characters) + instant ``YYYY-MM-DDTHH:MM:SS.ffffffZ`` (27 characters, always UTC) + +Both are fixed width with ASCII digits in fixed positions, so byte-lexicographic order +is chronological order. That is what lets containment and overlap be plain string +comparisons with identical SQL text in either dialect. + +The two axes never mix and never convert into one another. A date bound is a calendar +date: it acquires no time of day and no timezone, ever. An instant bound names a moment +and is normalized to UTC, so two instants written in different offsets compare as the +instants they name. A timestamp written without an offset is *read as UTC*, which is +the convention the rest of the codebase already uses for naive datetimes +(`utils.ensure_timezone_aware`, `recent_activity`). + +Two authored surfaces reach these values, and they trade precision for convenience in +opposite directions: + +* A **range literal** (`[2026-06-10,2026-07-27)`) is the precise form. Its bounds must + be written in the canonical lexical shapes above, to at most microsecond precision. +* A **point** (`2026-06-10`, `2026-06`, `2026`, `yesterday`) is the convenient form. It + denotes the span its precision covers, so an author never has to spell out a range to + say when something started. A point written in ISO calendar syntax is read literally, + because its text fixes its meaning; any other spelling is read with `dateparser`, + because there is no literal reading for a guess to contradict. +""" + +import re +from dataclasses import dataclass +from datetime import UTC, date, datetime, timedelta +from enum import StrEnum +from functools import lru_cache +from typing import TYPE_CHECKING, Any, Literal, assert_never, override + +if TYPE_CHECKING: # pragma: no cover - import exists only for the annotation below + from dateparser.date import DateDataParser + + +class TemporalQualifierError(ValueError): + """A temporal qualifier, range literal, or bound failed to parse or validate.""" + + +class TimeKind(StrEnum): + """Which kind of time an assertion describes. + + `recorded` is deliberately absent: recorded time is never authored in markdown. + """ + + EFFECTIVE = "effective" + VALID = "valid" + OCCURRED = "occurred" + DUE = "due" + MENTIONED = "mentioned" + + +class TemporalRangeAxis(StrEnum): + """Whether a range is measured in calendar dates or in instants.""" + + DATE = "date" + INSTANT = "instant" + + +EMPTY_RANGE_LITERAL = "empty" +OBSERVATION_EXTRACTOR = "observation" + +# Which component a slash-formatted date leads with. Only ambiguous forms consult it: +# `10/07/2026` is July 10 under YMD/DMY and October 7 under MDY, while `2026-06-10` is +# ISO and is never re-guessed. Mirrored by `BasicMemoryConfig.date_order`. +type DateOrder = Literal["YMD", "DMY", "MDY"] + +DEFAULT_DATE_ORDER: DateOrder = "YMD" + + +# --- Bound grammar --- + +# A date bound is exactly the canonical form, so authored and canonical text agree. +# The anchored pattern also rejects the compact `20260610` shape that +# `date.fromisoformat` accepts on 3.11+, which would break fixed-width ordering. +_DATE_BOUND = re.compile(r"^\d{4}-\d{2}-\d{2}$") + +# Sub-microsecond precision is refused rather than truncated: silently dropping digits +# would make the stored bound name a different instant than the author wrote. The +# offset is optional because a naive timestamp is read as UTC, not rejected. +_INSTANT_BOUND = re.compile( + r"^\d{4}-\d{2}-\d{2}[Tt]\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?(?:[Zz]|[+-]\d{2}:\d{2})?$" +) +_CANONICAL_INSTANT = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$") + +# Anything shaped like a date followed by a time separator is *meant* as a timestamp. +# Classifying it as an instant before validating it is what lets a broken timestamp +# report itself as one instead of as "not a calendar date". +_TIMESTAMP_SHAPE = re.compile(r"^\d{4}-\d{2}-\d{2}[Tt ]") + +# `[lower,upper)` and friends. Bounds carry no brackets and no comma, so one anchored +# pattern splits the literal without any nesting rules. +_RANGE_LITERAL = re.compile(r"^([\[(])([^,\[\]()]*),([^,\[\]()]*)([\])])$") + + +def _classify_bound(bound: str) -> TemporalRangeAxis: + """Decide which axis an authored bound is written on.""" + if _TIMESTAMP_SHAPE.match(bound): + return TemporalRangeAxis.INSTANT + return TemporalRangeAxis.DATE + + +def _canonical_date(bound: str) -> str: + if not _DATE_BOUND.match(bound): + raise TemporalQualifierError(f"date bound must be YYYY-MM-DD: {bound!r}") + try: + return date.fromisoformat(bound).isoformat() + except ValueError as exc: + raise TemporalQualifierError(f"not a calendar date: {bound!r}") from exc + + +def _instant_value(moment: datetime) -> str | None: + """Render one moment as the canonical fixed-width UTC instant. + + A naive moment is read as UTC rather than refused. That is the house convention for + every other naive datetime in the codebase, and it is what lets an author write + `2026-07-27T18:42:00` without learning RFC 3339's offset syntax first. + + None means the moment has no UTC rendering: shifting it by its offset carries it off + the calendar, as `9999-12-31T23:59:59-05:00` does into year 10000. Reported the way + `_next_calendar_day` reports its own edge -- each caller decides what running off the + calendar means for it -- rather than raised, so the overflow can never escape as a + bare `OverflowError` and fail a whole note's parse. + """ + if moment.tzinfo is None: + moment = moment.replace(tzinfo=UTC) + try: + utc = moment.astimezone(UTC) + except OverflowError: + return None + return utc.strftime("%Y-%m-%dT%H:%M:%S.%f") + "Z" + + +def _canonical_instant(bound: str) -> str: + if not _INSTANT_BOUND.match(bound): + raise TemporalQualifierError( + f"timestamp bound must be RFC 3339 to microsecond precision, " + f"with an optional offset or Z: {bound!r}" + ) + # RFC 3339 allows lowercase `t`/`z`, which `datetime.fromisoformat` rejects. Every + # other character in a matched bound is a digit or punctuation, so upper-casing the + # whole bound only touches those two markers. + try: + moment = datetime.fromisoformat(bound.upper()) + except ValueError as exc: + raise TemporalQualifierError(f"not a valid timestamp: {bound!r}") from exc + value = _instant_value(moment) + if value is None: + raise TemporalQualifierError( + f"timestamp bound leaves the calendar when converted to UTC: {bound!r}" + ) + return value + + +def canonical_bound(bound: str, axis: TemporalRangeAxis) -> str: + """Normalize one authored bound to the canonical fixed-width form for its axis.""" + if axis is TemporalRangeAxis.DATE: + return _canonical_date(bound) + return _canonical_instant(bound) + + +def _require_canonical(value: str, axis: TemporalRangeAxis) -> None: + """Reject a value that skipped `canonical_bound` on its way into a domain value.""" + pattern = _DATE_BOUND if axis is TemporalRangeAxis.DATE else _CANONICAL_INSTANT + if not pattern.match(value): + raise TemporalQualifierError(f"{axis.value} bound is not canonical: {value!r}") + + +def _next_calendar_day(bound: str) -> str | None: + """The canonical date after `bound`, or None when the calendar has none. + + Only `9999-12-31` has no successor. Reporting that as None rather than raising lets + each side of a range decide what running off the end of the calendar means for it: + an upper end there covers every remaining day, a lower end past it covers none. + """ + day = date.fromisoformat(bound) + if day == date.max: + return None + return (day + timedelta(days=1)).isoformat() + + +# --- Values --- + + +@dataclass(frozen=True, slots=True) +class TemporalPoint: + """One calendar date or instant that a containment question is asked about.""" + + axis: TemporalRangeAxis + value: str + + def __post_init__(self) -> None: + _require_canonical(self.value, self.axis) + + @override + def __str__(self) -> str: + return self.value + + +@dataclass(frozen=True, slots=True) +class TemporalRange: + """One authored interval on a single time axis. + + Bounds are canonical lexical strings; `None` means unbounded on that side. + Construction normalizes three PostgreSQL rules so no caller has to remember them: + an unbounded side is always exclusive, an interval containing no points *is* the + empty range, and -- exactly as `daterange` does -- a **date** range is rewritten + into the half-open `[lower,upper)` form. + + That last rule is what makes the scalar SQL predicate correct rather than merely + tidy. Calendar dates are a *discrete* domain, so `[a,b]` and `[a,b+1)` denote the + same set of days, but only the half-open spelling lets endpoint comparisons decide + membership. Left as authored, `(2026-01-01,2026-01-03)` holds only January 2 and + `(2026-01-02,2026-01-04)` holds only January 3 -- disjoint sets -- yet each raw + endpoint lies inside the other's bounds, so a comparison of raw endpoints reports + an overlap that does not exist. Canonicalized to `[2026-01-02,2026-01-03)` and + `[2026-01-03,2026-01-04)`, the same comparison is right. + + Instants are a continuous domain -- no moment is "the next one" -- so an instant + range keeps the inclusivity the author wrote and is never rewritten this way. + + Canonicalization changes the *stored* spelling, never the set of times: `[a,a]` + becomes `[a,a+1)`, the one day `a`. What the author typed is not lost; it is kept + verbatim on `TemporalAssertion.source_text`, which is what serialization replays + and what a search result quotes back. `__str__` renders the canonical form, and + re-parsing that rendering yields this same value. + """ + + axis: TemporalRangeAxis + lower: str | None = None + upper: str | None = None + lower_inclusive: bool = False + upper_inclusive: bool = False + is_empty: bool = False + + def __post_init__(self) -> None: + if self.is_empty: + # The empty range has no endpoints at all, so inclusivity is meaningless + # for it; representing it two ways would make equality lie. + if ( + self.lower is not None + or self.upper is not None + or self.lower_inclusive + or self.upper_inclusive + ): + raise TemporalQualifierError("the empty range carries no bounds") + return + + for bound in (self.lower, self.upper): + if bound is not None: + _require_canonical(bound, self.axis) + + # Canonical bounds are fixed width, so string order is chronological order. + # Judged on the bounds as authored: an interval written backwards is an author + # error to report, not an empty range to accept silently. + if self.lower is not None and self.upper is not None and self.lower > self.upper: + raise TemporalQualifierError( + f"range lower bound {self.lower} is after upper bound {self.upper}" + ) + + # PostgreSQL: an unbounded side cannot be inclusive; there is no endpoint. + if self.lower is None: + object.__setattr__(self, "lower_inclusive", False) + if self.upper is None: + object.__setattr__(self, "upper_inclusive", False) + + # --- Discrete canonical form --- + # + # Rewrite a date range to `[lower,upper)`. See the class docstring for why the + # scalar overlap predicate needs this and why instants must not get it. + if self.axis is TemporalRangeAxis.DATE: + if self.lower is not None and not self.lower_inclusive: + after_lower = _next_calendar_day(self.lower) + if after_lower is None: + # Nothing follows 9999-12-31, so a range starting strictly after it + # admits no date at all. + self._become_empty() + return + object.__setattr__(self, "lower", after_lower) + object.__setattr__(self, "lower_inclusive", True) + if self.upper is not None and self.upper_inclusive: + # None here loses no days: 9999-12-31 is the last date there is, so + # "through 9999-12-31 inclusive" and "unbounded above" hold the same + # set, and only the latter is representable in the canonical form. + object.__setattr__(self, "upper", _next_calendar_day(self.upper)) + object.__setattr__(self, "upper_inclusive", False) + + # PostgreSQL: an interval that admits no point at all *is* the empty range. The + # endpoints coincide without both being owned (`[a,a)`), or -- only reachable + # after the rewrite above, from `(a,a)` -- the lower end has overshot the upper. + if self.lower is not None and self.upper is not None: + admits_no_date = self.lower > self.upper or ( + self.lower == self.upper and not (self.lower_inclusive and self.upper_inclusive) + ) + if admits_no_date: + self._become_empty() + + def _become_empty(self) -> None: + """Collapse to the one empty representation, whatever bounds were written.""" + object.__setattr__(self, "lower", None) + object.__setattr__(self, "upper", None) + object.__setattr__(self, "lower_inclusive", False) + object.__setattr__(self, "upper_inclusive", False) + object.__setattr__(self, "is_empty", True) + + @classmethod + def empty(cls, axis: TemporalRangeAxis) -> "TemporalRange": + """The empty range on one axis.""" + return cls(axis=axis, is_empty=True) + + @override + def __str__(self) -> str: + """Render the canonical PostgreSQL range literal. + + This is the normalized interval, not the author's text -- a date range always + renders half-open. Feeding the result back to `parse_range_literal` reproduces + this same value, so the rendering is a fixed point rather than a lossy view. + """ + if self.is_empty: + return EMPTY_RANGE_LITERAL + lower = "" if self.lower is None else self.lower + upper = "" if self.upper is None else self.upper + return ( + f"{'[' if self.lower_inclusive else '('}{lower},{upper}" + f"{']' if self.upper_inclusive else ')'}" + ) + + +@dataclass(frozen=True, slots=True) +class TemporalFilter: + """A valid-time question asked of the stored assertions. + + Exactly one of `at` (containment) or `overlaps` may be given, or neither -- a + kind-only filter asks for sources that carry *any* assertion of that kind, which + is a legal and useful question. A filter that asks nothing at all is refused + rather than silently matching everything. + """ + + kind: TimeKind | None = None + at: TemporalPoint | None = None + overlaps: TemporalRange | None = None + + def __post_init__(self) -> None: + if self.at is not None and self.overlaps is not None: + raise TemporalQualifierError( + "a temporal filter asks either 'at' or 'overlaps', never both" + ) + if self.kind is None and self.at is None and self.overlaps is None: + raise TemporalQualifierError("a temporal filter must name a kind, a point, or a range") + + @property + def window(self) -> TemporalRange | None: + """The interval this filter tests against, or None for a kind-only filter. + + Containment of a point is overlap with the closed range `[p,p]`: both ask + whether the stored interval and the queried interval share at least one point. + Collapsing them here lets one predicate answer both questions, which is also + why the two can never disagree about inclusivity or bounds. On the date axis + `TemporalRange` canonicalizes that window to `[p,p+1)` -- still the single day + `p`, now in the half-open form the predicate compares correctly. + """ + if self.at is not None: + return TemporalRange( + axis=self.at.axis, + lower=self.at.value, + upper=self.at.value, + lower_inclusive=True, + upper_inclusive=True, + ) + return self.overlaps + + +@dataclass(frozen=True, slots=True) +class TemporalAssertion: + """One authored statement that a source is valid over a span of time. + + Source identity -- entity, source type, source row id -- is deliberately absent. + The parser reads markdown, where those ids do not exist yet; the projection layer + pairs this value with them when it writes derived rows. + + `source_text` is the exact authored token. Serialization replays it verbatim, so a + parse/serialize round trip reproduces the author's bounds and precision even though + `valid_during` holds the normalized form. + """ + + time_kind: TimeKind + valid_during: TemporalRange + source_text: str + extractor: str = OBSERVATION_EXTRACTOR + metadata: dict[str, Any] | None = None + + +# --- Literal parsing --- + + +def parse_range_literal(literal: str, *, axis: TemporalRangeAxis | None = None) -> TemporalRange: + """Parse a PostgreSQL-style range literal into a canonical `TemporalRange`. + + Accepts `[lower,upper)`, `(lower,upper]`, `[lower,)`, `(,upper)`, `(,)`, and the + bare token `empty`. `axis` asserts the axis the caller expects; when omitted it is + inferred from the bounds, which is why the bound-less forms require it explicitly. + """ + text = literal.strip() + if text == EMPTY_RANGE_LITERAL: + if axis is None: + raise TemporalQualifierError( + "the 'empty' range literal has no bounds, so its axis must be given" + ) + return TemporalRange.empty(axis) + + match = _RANGE_LITERAL.match(text) + if match is None: + raise TemporalQualifierError( + f"range literal must be [lower,upper), (lower,upper], or 'empty': {literal!r}" + ) + open_bracket, lower_text, upper_text, close_bracket = match.groups() + lower_text = lower_text.strip() + upper_text = upper_text.strip() + + written_axes = {_classify_bound(bound) for bound in (lower_text, upper_text) if bound} + if len(written_axes) > 1: + raise TemporalQualifierError( + f"a range must not mix date-only and timestamp bounds: {literal!r}" + ) + if not written_axes: + if axis is None: + raise TemporalQualifierError( + f"a fully unbounded range has no bounds to classify: {literal!r}" + ) + range_axis = axis + else: + range_axis = written_axes.pop() + if axis is not None and range_axis is not axis: + raise TemporalQualifierError( + f"expected {axis.value} bounds but found {range_axis.value} bounds: {literal!r}" + ) + + return TemporalRange( + axis=range_axis, + lower=canonical_bound(lower_text, range_axis) if lower_text else None, + upper=canonical_bound(upper_text, range_axis) if upper_text else None, + lower_inclusive=open_bracket == "[", + upper_inclusive=close_bracket == "]", + ) + + +def parse_point(text: str) -> TemporalPoint: + """Parse one authored date or timestamp into a canonical `TemporalPoint`.""" + bound = text.strip() + if not bound: + raise TemporalQualifierError("a temporal point must not be empty") + axis = _classify_bound(bound) + return TemporalPoint(axis=axis, value=canonical_bound(bound, axis)) + + +def parse_temporal_filter( + *, + valid_at: str | None = None, + valid_overlaps: str | None = None, + time_kind: str | None = None, +) -> TemporalFilter | None: + """Parse the three flat boundary fields into one portable filter value. + + Every request surface -- HTTP, MCP, CLI -- carries a valid-time question as these + three independent strings, so this is the one place that turns them into the domain + value. Sharing it is what lets a caller validate the question *before* asking it and + be certain the answer to "is this filter well formed?" is the same one the search + service will reach. + + Every rejection is deliberate and loud: an unknown kind, a malformed range literal, a + range mixing calendar dates with instants, or an impossible range raises rather than + degrading into a filter that quietly matches something else. A timestamp written + without an offset is not a rejection -- like every other naive datetime in the + codebase, it is read as UTC. + + Returns None when no valid-time question was asked at all. + """ + if not (valid_at or valid_overlaps or time_kind): + return None + + kind: TimeKind | None = None + if time_kind: + try: + kind = TimeKind(time_kind) + except ValueError as exc: + raise TemporalQualifierError( + f"unknown time_kind {time_kind!r}; expected one of " + f"{', '.join(item.value for item in TimeKind)}" + ) from exc + + return TemporalFilter( + kind=kind, + at=parse_point(valid_at) if valid_at else None, + overlaps=parse_range_literal(valid_overlaps) if valid_overlaps else None, + ) + + +# --- Flexible authored points --- + + +@lru_cache(maxsize=8) +def _date_data_parser(date_order: DateOrder) -> "DateDataParser": + """The flexible reader for authored points, built once per configured date order. + + Deferred import: dateparser costs ~0.13s and loads locale data, and the modules + that carry these values are imported on every CLI start (#886). Only an + observation that already looks like a qualifier ever reaches this function. + """ + from dateparser.date import DateDataParser + + return DateDataParser( + settings={ + "DATE_ORDER": date_order, + # Makes `period` report "time" when the author wrote a clock reading, + # which is exactly the date-vs-instant distinction this module keeps. + "RETURN_TIME_AS_PERIOD": True, + } + ) + + +def _next_month_start(year: int, month: int) -> date | None: + """The first day of the month after `year`-`month`, or None past the calendar's end. + + Only December 9999 has no successor month; year 10000 is not a date `datetime` can + hold. Reported as None for the same reason `_next_calendar_day` reports its own + edge: the caller decides what running off the end of the calendar means for it. + """ + if month < 12: + return date(year, month + 1, 1) + if year == date.max.year: + return None + return date(year + 1, 1, 1) + + +def _calendar_span(lower: date, upper: date | None) -> TemporalRange: + """The half-open calendar period `[lower,upper)`, unbounded when it runs to the end. + + A period whose successor is off the calendar needs no upper end: nothing follows + 9999-12-31, so `[lower,)` holds exactly the days `[lower,successor)` would have. It + is the same equivalence `TemporalRange` applies to an inclusive upper bound on the + last date, and it is why December 9999 is a period this reader can express rather + than one it fails on. + """ + return TemporalRange( + axis=TemporalRangeAxis.DATE, + lower=lower.isoformat(), + upper=None if upper is None else upper.isoformat(), + lower_inclusive=True, + ) + + +# --- Which language an authored point is written in --- +# +# An author writes a point in one of two languages, and they come with opposite promises. +# **ISO calendar syntax** is machine syntax: the text fixes the meaning, so it must be read +# literally or refused. **Everything else** -- `June 10, 2026`, `2026/03/04`, `10/07/2026`, +# `yesterday` -- is human syntax with no literal reading to contradict, so the flexible +# reader is trusted with it. +# +# The variants below are what a point can be once that question is settled, and settling it +# *once* is the whole design. Four review rounds went the other way: each added a shape test +# whose failure meant "not my business", so a token that failed the test fell through to the +# flexible reader and the next round found another shape that failed it. Here the classifier +# is total: a token that opens with ISO syntax is an `_IsoDay`, an `_IsoMonth` or a +# `_MalformedIso`, and there is no fourth answer to fall through on. +# +# What that buys is narrower than "ISO-shaped text never reaches the flexible reader", and +# stating it precisely matters, because the loose version is false. An `_IsoDay`'s *trailing* +# text is still read by the flexible reader -- that is what reads `2026-06-10 10:00 AM`, and +# no grammar of clock spellings could. What the classifier settles for good is the *calendar*: +# a head that names no date dies here, and a real one is carried on the variant so the reading +# below can be held to it. The trailing is fenced by two rules instead, and dateparser's answer +# is believed only when both hold. It must come back as a time of day on the day the head names +# -- checked in `_read_iso_day`, against what it *returned*, since a suffix's looks do not say +# what it will do with it. And the text must not spell precision a canonical instant cannot +# carry -- checked here, on the text, because that is the one defect the returned-value check +# cannot see: a truncated fraction still lands on the right day. + +# The ISO calendar components a point *opens* with: `YYYY-MM` and an optional `-DD`. A date +# carrying a time (`2026-06-10T14:00`, `2026-06-10 10:00 AM`) is matched on its date part +# alone, because `\d+` cannot cross the separator -- the rest is `trailing`, judged below. +# +# Each component is `\d+` rather than `\d{2}`, and nothing terminates the pattern, so the +# head matches whenever a point opens with ISO syntax at all. Both rules exist because the +# earlier cuts of this guard failed to match a malformed token and so let it escape: against +# `\d{2}` the day of `2026-01-0100` left a trailing `00` and matched nothing, and against a +# trailing `(?![\d-])` lookahead `2026-01-01-` matched nothing. Both reached the flexible +# reader, which is the one outcome ISO syntax must never have. +_ISO_CALENDAR_HEAD = re.compile(r"^(\d{4})-(\d+)(?:-(\d+))?") + +# A fractional-second run too wide for a canonical instant to carry. `_INSTANT_BOUND` caps the +# fraction at six digits and *refuses* a longer one rather than truncating it, because dropping +# digits would store a different instant than the author wrote -- but that refusal only ever +# governed the strict path. The flexible reader has no such scruple: it truncates +# `2026-01-01T10:00:00.1234567` to `...123456Z` and reports a time on the right day, so every +# check `_read_iso_day` makes passes and the authored instant is quietly rewritten on each +# reindex. Judged on the text so both paths refuse the same token for the same reason, and it +# is the same reason the calendar width rule exists: a digit run wider than the syntax allows +# is a typo, not a shorthand. Six digits and fewer are untouched -- `14:00:00.5` is precision +# a canonical instant holds exactly, so it still reads. +_OVER_PRECISE_FRACTION = re.compile(r"\.\d{7,}") + + +def _named_calendar_date(year: str, month: str, day: str | None) -> date | None: + """The date ISO-shaped calendar components name, or None when they name none. + + A month-only head is placed on the first of that month: the day is a component the + author did not write, not one to guess at. `date` is the authority rather than a range + check because it already owns leap years and month lengths. + """ + # A month or a day is written with one or two digits, and that width is what separates + # an author's shorthand from an author's typo: `2026-1-5` is a legitimate unpadded + # spelling of a real date, while the `0100` in `2026-01-0100` is no day at all. Judged + # before `date`, which takes a C long and raises OverflowError -- not the ValueError + # below -- once a run of digits grows past it. + if len(month) > 2 or (day is not None and len(day) > 2): + return None + try: + return date(int(year), int(month), 1 if day is None else int(day)) + except ValueError: + return None + + +@dataclass(frozen=True, slots=True) +class _IsoDay: + """A point whose ISO head names a calendar day, and whatever was written after it. + + The day is authoritative: it is what the author typed, so no reading of `trailing` may + contradict it. `trailing` is empty for a bare date; when it is not, the point is an + instant, because a time of day is the only thing that can follow a complete date. + """ + + day: date + trailing: str + + +@dataclass(frozen=True, slots=True) +class _IsoMonth: + """A point whose ISO head names a calendar month (`2026-06`), and so denotes it. + + There is deliberately nowhere to put trailing text: nothing may follow a month. A clock + reading needs a day to fall on, and the flexible reader supplies the day it was not + given from *today*, so `2026-06 10:00` read as June 7 in March and June 1 in September + -- the same note projecting different valid time on different indexing days. + """ + + year: int + month: int + + +@dataclass(frozen=True, slots=True) +class _MalformedIso: + """A point written in ISO syntax that cannot be read as written. + + `2026-13-01`, `2026-01-0100`, `2026-06 10:00`, `2026-01-01T10:00:00.1234567`. Either the + components name nothing on the calendar, or they name a moment finer than a canonical + instant records. The author reached for a machine date and missed, so there is no reading + to fall back on -- only a guess, which is what this variant exists to make unreachable. + """ + + +@dataclass(frozen=True, slots=True) +class _FlexiblePoint: + """A point in no machine syntax at all, for the flexible reader to interpret.""" + + +type _AuthoredPoint = _IsoDay | _IsoMonth | _MalformedIso | _FlexiblePoint + +_MALFORMED_ISO = _MalformedIso() +_FLEXIBLE_POINT = _FlexiblePoint() + + +def _classify_authored_point(point: str) -> _AuthoredPoint: + """Decide which language one authored point is written in, and what it names. + + Total by construction, which is the property the whole design rests on: opening with + ISO syntax settles the question, and the three ISO variants are all a token can then + be. There is no "looks ISO but is not this function's business" answer to fall through + on, which is what every earlier cut of this guard offered and what each review round + found another way to reach. + """ + head = _ISO_CALENDAR_HEAD.match(point) + if head is None: + return _FLEXIBLE_POINT + + year, month, day = head.groups() + named = _named_calendar_date(year, month, day) + if named is None: + return _MALFORMED_ISO + + trailing = point[head.end() :] + if day is None: + # Trigger: the head names a month, with or without text after it. + # Why: a month is a complete point on its own, so anything following it is part of + # a date this head cannot carry -- see `_IsoMonth` for what reading it costs. + # Outcome: a bare month denotes its own period; a month with anything after it is + # malformed. + return _MALFORMED_ISO if trailing else _IsoMonth(int(year), int(month)) + + # Trigger: the text after the date spells a fraction of a second wider than six digits. + # Why: no reader here can store it, and the two that try disagree -- `_canonical_instant` + # refuses it, while the flexible reader truncates it and still answers with a time on + # the head's day, which is precisely what `_read_iso_day`'s returned-value check cannot + # catch. A guard that asks what came back cannot see digits that never made it in. + # Outcome: refused as malformed, so the strict and flexible paths give the same answer to + # the same text and the token stays observation content rather than a rounded instant. + if _OVER_PRECISE_FRACTION.search(trailing): + return _MALFORMED_ISO + return _IsoDay(named, trailing) + + +def _read_iso_day(iso: _IsoDay, point: str, date_order: DateOrder) -> TemporalRange | None: + """Read a point whose head names a calendar day, holding it to its own text.""" + if not iso.trailing: + return TemporalRange( + axis=TemporalRangeAxis.DATE, lower=iso.day.isoformat(), lower_inclusive=True + ) + + if _INSTANT_BOUND.match(point): + # Trigger: the whole token is canonical RFC 3339. + # Why: the author wrote the one form this module defines exactly, so it is read + # exactly -- to the microsecond, and refused rather than rounded when it names no + # moment (`2026-06-10T25:00:00+02:00`) or leaves the calendar in UTC. The flexible + # reader is neither that precise nor that strict. + # Outcome: an instant, or a refusal; never a guess. + try: + instant = _canonical_instant(point) + except TemporalQualifierError: + return None + return TemporalRange(axis=TemporalRangeAxis.INSTANT, lower=instant, lower_inclusive=True) + + # The author wrote a clock reading in some spelling of their own, so the flexible reader + # is asked for it -- but only for it. What it hands back must be a time of day on the + # very day the head names, which is the check that keeps its guessing out of the answer: + # dateparser silently drops a suffix it cannot use (`2026-01-01T`, `2026-01-01Z`, + # `2026-01-01+14:00` all came back as the bare date), and a suffix it half-understands + # makes it abandon the ISO reading and re-guess the components under the configured + # order (`2026-06-10x` came back as October 6). Asking what it *returned* rather than + # what the suffix looks like is what covers every such shape, named or not. + date_data = _date_data_parser(date_order).get_date_data(point) + moment = date_data.date_obj + if moment is None or date_data.period != "time" or moment.date() != iso.day: + return None + instant = _instant_value(moment) + if instant is None: + # A moment that leaves the calendar in UTC names no storable instant, so it reads + # as no date at all -- the token stays content. + return None + return TemporalRange(axis=TemporalRangeAxis.INSTANT, lower=instant, lower_inclusive=True) + + +def _read_flexible_point(point: str, date_order: DateOrder) -> TemporalRange | None: + """Read a point written in no machine syntax, taking the flexible reader at its word.""" + date_data = _date_data_parser(date_order).get_date_data(point) + moment = date_data.date_obj + if moment is None: + return None + + # dateparser fills components the author did not write from today's date, so only + # the components `period` vouches for may be read off `moment`. + match date_data.period: + case "time": + instant = _instant_value(moment) + if instant is None: + # A moment that leaves the calendar in UTC names no storable instant, + # so it reads as no date at all -- the token stays content. + return None + return TemporalRange( + axis=TemporalRangeAxis.INSTANT, + lower=instant, + lower_inclusive=True, + ) + case "year": + # The month after December is the following January 1 -- except at year + # 9999, where there is none and `_calendar_span` leaves the span open at + # `[9999-01-01,)`, which is still exactly that year. + return _calendar_span(date(moment.year, 1, 1), _next_month_start(moment.year, 12)) + case "month": + return _calendar_span( + date(moment.year, moment.month, 1), + _next_month_start(moment.year, moment.month), + ) + case _: + # Day precision, and any coarser calendar period dateparser resolves to a + # specific day ("last week"): the day it named, onward. + return TemporalRange( + axis=TemporalRangeAxis.DATE, + lower=moment.date().isoformat(), + lower_inclusive=True, + ) + + +def parse_authored_point( + text: str, *, date_order: DateOrder = DEFAULT_DATE_ORDER +) -> TemporalRange | None: + """Read one authored point into the interval its precision denotes. + + The precision the author wrote is the meaning: + + 2026 -> [2026-01-01,2027-01-01) the year + 2026-06 -> [2026-06-01,2026-07-01) the month + 2026-06-10 -> [2026-06-10,) from that date onward + 2026-06-10T14:00:00 -> [that instant,) from that moment onward + + A year or a month is a period the author delimited by writing it. A date or a + moment is not: `@effective 2026-06-10` means the decision took effect that day and + still holds, so closing the range at midnight would expire it overnight. Callers + that need a closed interval write the range literal instead. + + Non-ISO spellings are read leniently, because guessing at `June 10, 2026` is the + whole point of this reader. A token that *is* ISO-shaped is held to its own text + instead: its calendar components must name a real date, and anything trailing them + must be a time of day on that date, written to a precision this module can store. + `2026-06-10 10:00 AM` reads; `2026-01-01T` does not, because the author reached for an + instant and no instant is there; `2026-01-01T10:00:00.1234567` does not either, + because storing it would mean dropping the digits that made it worth writing. + + Returns None when the text names no date. That is not an error -- the caller leaves + such a token as ordinary observation content. + """ + point = text.strip() + match _classify_authored_point(point): + case _IsoDay() as iso: + return _read_iso_day(iso, point, date_order) + case _IsoMonth() as iso: + return _calendar_span( + date(iso.year, iso.month, 1), _next_month_start(iso.year, iso.month) + ) + case _MalformedIso(): + # The author wrote a machine date that names nothing. Refusal is `None`, as + # everywhere else here: the token stays ordinary observation content, + # unindexed but still full-text searchable. + return None + case _FlexiblePoint(): + return _read_flexible_point(point, date_order) + case unreachable: # pragma: no cover - `_AuthoredPoint` is closed + assert_never(unreachable) diff --git a/tests/api/v2/test_search_router_telemetry.py b/tests/api/v2/test_search_router_telemetry.py index a3c33a0f1..d1eec98ba 100644 --- a/tests/api/v2/test_search_router_telemetry.py +++ b/tests/api/v2/test_search_router_telemetry.py @@ -32,7 +32,7 @@ def fake_span(name: str, **attrs): operations.append((name, attrs)) yield - async def fake_to_search_results(entity_service, results): + async def fake_to_search_results(entity_service, results, *, temporal_by_source=None): return [] monkeypatch.setattr(logfire, "span", fake_span) @@ -43,6 +43,9 @@ async def fake_to_search_results(entity_service, results): query=SearchQuery(text="hello world"), search_service=FakeSearchService(), entity_service=object(), + # This query carries no valid-time filter, so neither is touched. + temporal_repository=object(), + session_maker=object(), read_cache=None, response=http_response, project_id="11111111-1111-1111-1111-111111111111", @@ -64,5 +67,6 @@ async def fake_to_search_results(entity_service, results): "retrieval_mode": "fts", "has_query": True, "has_filters": False, + "has_temporal_filter": False, }, ) diff --git a/tests/api/v2/test_search_router_temporal.py b/tests/api/v2/test_search_router_temporal.py new file mode 100644 index 000000000..a49f83af0 --- /dev/null +++ b/tests/api/v2/test_search_router_temporal.py @@ -0,0 +1,291 @@ +"""Valid-time filters over the v2 search endpoint (SPEC-82). + +The router is where three things have to line up: the filter reaches the service, the +matched assertions come back with the results, and the response says the filter actually +ran. That last one is not decoration -- `SearchQuery` ignores unknown fields, so without +an explicit confirmation an older server would answer a valid-time query with unfiltered +results that look filtered. +""" + +from textwrap import dedent +from typing import Any + +import pytest +from httpx import AsyncClient + +from basic_memory.models import Project +from basic_memory.schemas import Entity as EntitySchema + +CACHE_LAYER_MARKDOWN = dedent(""" + # Cache Layer + + ## Observations + - [decision] @effective[2026-06-10,2026-07-27) The cache layer will use Redis. + - [decision] @effective[2026-07-27,) The cache layer will use Memcached. + """) + +UNDATED_MARKDOWN = dedent(""" + # Queue Layer + + ## Observations + - [decision] The queue layer will use RabbitMQ. + """) + + +async def _index_note(entity_service, search_service, title: str, content: str): + entity, _ = await entity_service.create_or_update_entity( + EntitySchema( + title=title, + note_type="note", + directory="decisions", + content=content, + ) + ) + await search_service.index_entity(entity) + return entity + + +async def _search(client: AsyncClient, v2_project_url: str, **query: Any) -> dict[str, Any]: + response = await client.post(f"{v2_project_url}/search/", json=query) + assert response.status_code == 200, response.text + return response.json() + + +@pytest.mark.asyncio +async def test_temporal_filter_round_trips_through_v2_search( + client: AsyncClient, + test_project: Project, + v2_project_url: str, + entity_service, + search_service, +): + """A valid-time query narrows to the observation in force and explains why.""" + await _index_note(entity_service, search_service, "Cache Layer", CACHE_LAYER_MARKDOWN) + + payload = await _search( + client, + v2_project_url, + text="cache layer", + entity_types=["observation"], + time_kind="effective", + valid_at="2026-07-28", + ) + + assert payload["temporal_applied"] is True + contents = [result["content"] for result in payload["results"]] + assert any("Memcached" in (content or "") for content in contents), contents + assert not any("Redis" in (content or "") for content in contents), contents + + [result] = payload["results"] + [assertion] = result["temporal"] + assert assertion["kind"] == "effective" + assert assertion["source_text"] == "@effective[2026-07-27,)" + assert assertion["valid_during"] == { + "axis": "date", + "literal": "[2026-07-27,)", + "lower": "2026-07-27", + "upper": None, + "lower_inclusive": True, + "upper_inclusive": False, + "is_empty": False, + } + + +@pytest.mark.asyncio +async def test_overlap_filter_returns_both_competing_decisions( + client: AsyncClient, + test_project: Project, + v2_project_url: str, + entity_service, + search_service, +): + """A window spanning the cutover overlaps both effective periods.""" + await _index_note(entity_service, search_service, "Cache Layer", CACHE_LAYER_MARKDOWN) + + payload = await _search( + client, + v2_project_url, + text="cache layer", + entity_types=["observation"], + valid_overlaps="[2026-06-01,2026-08-01)", + ) + + assert payload["temporal_applied"] is True + contents = " ".join(result["content"] or "" for result in payload["results"]) + assert "Redis" in contents and "Memcached" in contents + + +@pytest.mark.asyncio +async def test_search_without_a_temporal_filter_is_unchanged( + client: AsyncClient, + test_project: Project, + v2_project_url: str, + entity_service, + search_service, +): + """An ordinary search payload is byte-for-byte what it was before valid time. + + `temporal_applied` stays null rather than false, and no result carries a temporal + block, so nothing about an existing client's parsing changes. + """ + await _index_note(entity_service, search_service, "Cache Layer", CACHE_LAYER_MARKDOWN) + + payload = await _search( + client, v2_project_url, text="cache layer", entity_types=["observation"] + ) + + assert payload["temporal_applied"] is None + assert payload["results"] + assert all(result["temporal"] is None for result in payload["results"]) + + +@pytest.mark.asyncio +async def test_undated_note_is_excluded_and_the_exclusion_is_confirmed( + client: AsyncClient, + test_project: Project, + v2_project_url: str, + entity_service, + search_service, +): + """Acceptance 8 over HTTP: undated sources drop out, and the server says so.""" + await _index_note(entity_service, search_service, "Cache Layer", CACHE_LAYER_MARKDOWN) + await _index_note(entity_service, search_service, "Queue Layer", UNDATED_MARKDOWN) + + unfiltered = await _search(client, v2_project_url, text="layer", entity_types=["observation"]) + assert any("RabbitMQ" in (r["content"] or "") for r in unfiltered["results"]) + + filtered = await _search( + client, + v2_project_url, + text="layer", + entity_types=["observation"], + valid_at="2026-07-28", + ) + assert filtered["temporal_applied"] is True + assert not any("RabbitMQ" in (r["content"] or "") for r in filtered["results"]) + + +@pytest.mark.asyncio +async def test_pagination_totals_respect_the_temporal_filter( + client: AsyncClient, + test_project: Project, + v2_project_url: str, + entity_service, + search_service, +): + """`total` comes from a separate count query; it must run the same predicate. + + The router derives `has_more` from that total, so a count that ignored valid time + would advertise pages that do not exist. + """ + await _index_note(entity_service, search_service, "Cache Layer", CACHE_LAYER_MARKDOWN) + await _index_note(entity_service, search_service, "Queue Layer", UNDATED_MARKDOWN) + + payload = await _search( + client, + v2_project_url, + text="layer", + entity_types=["observation"], + valid_at="2026-07-28", + ) + + assert payload["total"] == len(payload["results"]) == 1 + assert payload["has_more"] is False + + +@pytest.mark.asyncio +async def test_a_valid_time_query_with_no_matches_still_confirms_the_filter( + client: AsyncClient, + test_project: Project, + v2_project_url: str, + entity_service, + search_service, +): + """An empty answer to a valid-time question is different from an unfiltered one. + + Nothing was in force in 2020, so there is nothing to hydrate -- but the caller still + needs to know the filter ran, or it cannot tell this apart from a stale server. + """ + await _index_note(entity_service, search_service, "Cache Layer", CACHE_LAYER_MARKDOWN) + + payload = await _search( + client, + v2_project_url, + text="cache layer", + entity_types=["observation"], + valid_at="2020-01-01", + ) + + assert payload["results"] == [] + assert payload["total"] == 0 + assert payload["temporal_applied"] is True + + +@pytest.mark.asyncio +async def test_valid_at_and_valid_overlaps_together_are_rejected( + client: AsyncClient, + test_project: Project, + v2_project_url: str, +): + """The schema refuses the contradictory pair, so it never reaches the service.""" + response = await client.post( + f"{v2_project_url}/search/", + json={"text": "cache", "valid_at": "2026-07-28", "valid_overlaps": "[2026-06-10,)"}, + ) + + assert response.status_code == 422 + assert "not both" in response.text + + +@pytest.mark.asyncio +async def test_temporal_only_query_is_accepted_as_criteria( + client: AsyncClient, + test_project: Project, + v2_project_url: str, + entity_service, + search_service, +): + """A valid-time filter alone is a complete search request.""" + await _index_note(entity_service, search_service, "Cache Layer", CACHE_LAYER_MARKDOWN) + + payload = await _search( + client, v2_project_url, entity_types=["observation"], time_kind="effective" + ) + + assert payload["temporal_applied"] is True + assert len(payload["results"]) == 2 + + +@pytest.mark.asyncio +async def test_read_cache_distinguishes_two_valid_time_questions( + client: AsyncClient, + test_project: Project, + v2_project_url: str, + entity_service, + search_service, +): + """The response cache keys on the whole query, so two dates cannot share an entry. + + The digest hashes `SearchQuery.model_dump()`, which now includes the valid-time + fields; without that, the second question would be answered with the first's cached + results. + """ + await _index_note(entity_service, search_service, "Cache Layer", CACHE_LAYER_MARKDOWN) + + after = await _search( + client, + v2_project_url, + text="cache layer", + entity_types=["observation"], + valid_at="2026-07-28", + ) + before = await _search( + client, + v2_project_url, + text="cache layer", + entity_types=["observation"], + valid_at="2026-07-01", + ) + + assert "Memcached" in (after["results"][0]["content"] or "") + assert "Redis" in (before["results"][0]["content"] or "") diff --git a/tests/cloud/test_cloud_services.py b/tests/cloud/test_cloud_services.py index a4097dfac..2cfd4395f 100644 --- a/tests/cloud/test_cloud_services.py +++ b/tests/cloud/test_cloud_services.py @@ -523,6 +523,7 @@ def __call__(self) -> RecordingSession: observation_repository = object() section_repository = object() + temporal_repository = object() relation_repository = object() class WriteRepositories: @@ -536,6 +537,11 @@ def section_repository(self, project_id: int) -> object: events.append("section_repository") return section_repository + def temporal_repository(self, project_id: int) -> object: + assert project_id == publication.project_id + events.append("temporal_repository") + return temporal_repository + def relation_repository(self, project_id: int) -> object: assert project_id == publication.project_id events.append("relation_repository") @@ -565,11 +571,13 @@ def __init__( *, observation_repository: object, section_repository: object, + temporal_repository: object, relation_repository: object, session_maker: object, ) -> None: assert observation_repository is not None assert section_repository is not None + assert temporal_repository is not None assert relation_repository is not None assert session_maker is not None @@ -615,6 +623,7 @@ async def publish( "relation_repository", "observation_repository", "section_repository", + "temporal_repository", "publish", ] @@ -641,6 +650,10 @@ def section_repository(self, project_id: int) -> object: assert project_id == publication.project_id return object() + def temporal_repository(self, project_id: int) -> object: + assert project_id == publication.project_id + return object() + def relation_repository(self, project_id: int) -> object: assert project_id == publication.project_id return object() @@ -663,11 +676,13 @@ def __init__( *, observation_repository: object, section_repository: object, + temporal_repository: object, relation_repository: object, session_maker: object, ) -> None: assert observation_repository is not None assert section_repository is not None + assert temporal_repository is not None assert relation_repository is not None assert session_maker is not None diff --git a/tests/index/test_local_project_index.py b/tests/index/test_local_project_index.py index 5ea943fdd..5d3acc2c0 100644 --- a/tests/index/test_local_project_index.py +++ b/tests/index/test_local_project_index.py @@ -61,7 +61,11 @@ ) from basic_memory.indexing.relation_persistence import RelationGenerationPublisher from basic_memory.models import Entity, Project, Relation -from basic_memory.repository import EntityRepository, NoteSectionRepository +from basic_memory.repository import ( + EntityRepository, + MemoryTimeIndexRepository, + NoteSectionRepository, +) from basic_memory.repository.note_content_repository import ( AcceptedNoteContentWrite, NoteContentRepository, @@ -1803,6 +1807,7 @@ async def test_local_relation_resolution_refreshes_pending_source_without_markdo observation_repository=observation_repository, relation_repository=relation_repository, section_repository=NoteSectionRepository(project_id=observation_repository.project_id), + temporal_repository=MemoryTimeIndexRepository(project_id=observation_repository.project_id), session_maker=session_maker, ).publish( entity_id=source_id, diff --git a/tests/indexing/test_accepted_note_mutation_runner.py b/tests/indexing/test_accepted_note_mutation_runner.py index 7efa8fe52..31e9a9fe9 100644 --- a/tests/indexing/test_accepted_note_mutation_runner.py +++ b/tests/indexing/test_accepted_note_mutation_runner.py @@ -49,6 +49,10 @@ AcceptedSectionWrite, ) from basic_memory.repository.entity_repository import AcceptedPendingEntityWrite +from basic_memory.repository.memory_time_index_repository import ( + AcceptedTemporalAssertion, + TemporalGenerationWriteResult, +) from basic_memory.repository.note_section_repository import SectionGenerationWriteResult from basic_memory.repository.observation_repository import ObservationGenerationWriteResult from basic_memory.repository.relation_repository import RelationGenerationWriteResult @@ -551,6 +555,20 @@ async def replace_sections_for_generation( raise AssertionError("section publication was not expected inside the accepted transaction") +class _TemporalRepository: + async def replace_assertions_for_generation( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + assertions: Sequence[AcceptedTemporalAssertion], + ) -> TemporalGenerationWriteResult: + raise AssertionError( + "temporal publication was not expected inside the accepted transaction" + ) + + class _RelationRepository: def __init__(self) -> None: self.calls: list[tuple[int, Sequence[AcceptedRelationWrite]]] = [] @@ -595,6 +613,7 @@ class _MutationWriteRepositories: search_repository_result: _SearchRepository observation_repository_result: _ObservationRepository section_repository_result: _SectionRepository + temporal_repository_result: _TemporalRepository relation_repository_result: _RelationRepository def pending_entity_repository(self, project_id: int) -> _PendingEntityRepository: @@ -617,6 +636,10 @@ def section_repository(self, project_id: int) -> _SectionRepository: _ = project_id return self.section_repository_result + def temporal_repository(self, project_id: int) -> _TemporalRepository: + _ = project_id + return self.temporal_repository_result + def relation_repository(self, project_id: int) -> _RelationRepository: _ = project_id return self.relation_repository_result @@ -763,6 +786,7 @@ def _dependencies( search_repository_result=search_repository, observation_repository_result=observation_repository or _ObservationRepository(), section_repository_result=_SectionRepository(), + temporal_repository_result=_TemporalRepository(), relation_repository_result=relation_repository or _RelationRepository(), ), move_policy=move_policy diff --git a/tests/indexing/test_accepted_note_write_runner.py b/tests/indexing/test_accepted_note_write_runner.py index afb1b6bfe..b4276e19c 100644 --- a/tests/indexing/test_accepted_note_write_runner.py +++ b/tests/indexing/test_accepted_note_write_runner.py @@ -45,6 +45,10 @@ AcceptedRelationWrite, AcceptedSectionWrite, ) +from basic_memory.repository.memory_time_index_repository import ( + AcceptedTemporalAssertion, + TemporalGenerationWriteResult, +) from basic_memory.repository.note_section_repository import SectionGenerationWriteResult from basic_memory.repository.observation_repository import ObservationGenerationWriteResult from basic_memory.repository.relation_repository import RelationGenerationWriteResult @@ -166,6 +170,22 @@ async def replace_sections_for_generation( return SectionGenerationWriteResult(generation_is_current=True) +class _TemporalRepository: + def __init__(self) -> None: + self.calls: list[tuple[int, Sequence[AcceptedTemporalAssertion]]] = [] + + async def replace_assertions_for_generation( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + assertions: Sequence[AcceptedTemporalAssertion], + ) -> TemporalGenerationWriteResult: + self.calls.append((entity_id, assertions)) + return TemporalGenerationWriteResult(generation_is_current=True) + + class _RelationRepository: def __init__(self) -> None: self.calls: list[tuple[int, Sequence[AcceptedRelationWrite]]] = [] @@ -227,6 +247,10 @@ def section_repository(self, project_id: int) -> _SectionRepository: assert project_id == 7 return _SectionRepository() + def temporal_repository(self, project_id: int) -> _TemporalRepository: + assert project_id == 7 + return _TemporalRepository() + def relation_repository(self, project_id: int) -> _RelationRepository: assert project_id == 7 return _RelationRepository() @@ -238,6 +262,7 @@ def relation_repository(self, project_id: int) -> _RelationRepository: assert isinstance(repositories.search_repository(7), _SearchRepository) assert isinstance(repositories.observation_repository(7), _ObservationRepository) assert isinstance(repositories.section_repository(7), _SectionRepository) + assert isinstance(repositories.temporal_repository(7), _TemporalRepository) assert isinstance(repositories.relation_repository(7), _RelationRepository) @@ -434,6 +459,10 @@ def _unexpected_section_repository(_project_id: int) -> _SectionRepository: raise AssertionError("section repository was not expected") +def _unexpected_temporal_repository(_project_id: int) -> _TemporalRepository: + raise AssertionError("temporal repository was not expected") + + @dataclass(frozen=True, slots=True) class _RepositoryProvider: pending_entity_repository_result: _PendingEntityRepository | None = None @@ -441,6 +470,7 @@ class _RepositoryProvider: search_repository_result: _SearchRepository | None = None observation_repository_result: _ObservationRepository | None = None section_repository_result: _SectionRepository | None = None + temporal_repository_result: _TemporalRepository | None = None relation_repository_result: _RelationRepository | None = None def pending_entity_repository(self, project_id: int) -> _PendingEntityRepository: @@ -468,6 +498,11 @@ def section_repository(self, project_id: int) -> _SectionRepository: return _unexpected_section_repository(project_id) return self.section_repository_result + def temporal_repository(self, project_id: int) -> _TemporalRepository: + if self.temporal_repository_result is None: + return _unexpected_temporal_repository(project_id) + return self.temporal_repository_result + def relation_repository(self, project_id: int) -> _RelationRepository: if self.relation_repository_result is None: return _unexpected_relation_repository(project_id) @@ -481,6 +516,7 @@ def _repository_provider( search_repository: _SearchRepository | None = None, observation_repository: _ObservationRepository | None = None, section_repository: _SectionRepository | None = None, + temporal_repository: _TemporalRepository | None = None, relation_repository: _RelationRepository | None = None, ) -> AcceptedNoteWriteRepositories: """Build a fail-fast fake repository provider for one focused test.""" @@ -488,6 +524,7 @@ def _repository_provider( pending_entity_repository_result=pending_entity_repository, observation_repository_result=observation_repository, section_repository_result=section_repository, + temporal_repository_result=temporal_repository, relation_repository_result=relation_repository, note_content_repository_result=note_content_repository, search_repository_result=search_repository, diff --git a/tests/indexing/test_relation_persistence.py b/tests/indexing/test_relation_persistence.py index d6b8dfe28..144305f9a 100644 --- a/tests/indexing/test_relation_persistence.py +++ b/tests/indexing/test_relation_persistence.py @@ -24,6 +24,11 @@ AcceptedNoteContentWrite, NoteContentRepository, ) +from basic_memory.repository.memory_time_index_repository import ( + AcceptedTemporalAssertion, + MemoryTimeIndexRepository, + TemporalGenerationWriteResult, +) from basic_memory.repository.note_section_repository import ( AcceptedSectionWrite, NoteSectionRepository, @@ -107,7 +112,11 @@ async def replace_observations_for_generation( assert session is not None self.events.append("observations") self.calls.append((generation, tuple(observations))) - return ObservationGenerationWriteResult(generation_is_current=self.generation_is_current) + return ObservationGenerationWriteResult( + generation_is_current=self.generation_is_current, + # The real repository returns one freshly minted row id per observation. + observation_ids=tuple(range(1, len(observations) + 1)), + ) @dataclass(slots=True) @@ -132,6 +141,28 @@ async def replace_sections_for_generation( return SectionGenerationWriteResult(generation_is_current=self.generation_is_current) +@dataclass(slots=True) +class RecordingTemporalGenerationStore: + """Record the fenced temporal replacement produced by the publisher.""" + + generation_is_current: bool = True + events: list[str] = field(default_factory=list) + calls: list[tuple[int, tuple[AcceptedTemporalAssertion, ...]]] = field(default_factory=list) + + async def replace_assertions_for_generation( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + assertions: Sequence[AcceptedTemporalAssertion], + ) -> TemporalGenerationWriteResult: + assert session is not None + self.events.append("temporal") + self.calls.append((generation, tuple(assertions))) + return TemporalGenerationWriteResult(generation_is_current=self.generation_is_current) + + @pytest.mark.asyncio async def test_relation_generation_publisher_commits_sorted_chunks_before_cleanup( monkeypatch: pytest.MonkeyPatch, @@ -156,10 +187,12 @@ async def fake_scoped_session( store = RecordingRelationGenerationStore() observation_store = RecordingObservationGenerationStore(events=store.events) section_store = RecordingSectionGenerationStore(events=store.events) + temporal_store = RecordingTemporalGenerationStore(events=store.events) publisher = RelationGenerationPublisher( relation_repository=store, observation_repository=observation_store, section_repository=section_store, + temporal_repository=temporal_store, session_maker=cast(async_sessionmaker[AsyncSession], object()), ) relations = [ @@ -191,7 +224,15 @@ async def fake_scoped_session( ) assert generation_is_current - assert store.events == ["begin", "observations", "sections", "upsert", "upsert", "cleanup"] + assert store.events == [ + "begin", + "observations", + "temporal", + "sections", + "upsert", + "upsert", + "cleanup", + ] assert observation_store.calls == [ (7, (AcceptedObservationWrite("Observed", "note", None, ["graph"]),)) ] @@ -242,10 +283,12 @@ async def fake_scoped_session( store = RecordingRelationGenerationStore(generation_is_current=False) observation_store = RecordingObservationGenerationStore(events=store.events) section_store = RecordingSectionGenerationStore(events=store.events) + temporal_store = RecordingTemporalGenerationStore(events=store.events) publisher = RelationGenerationPublisher( relation_repository=store, observation_repository=observation_store, section_repository=section_store, + temporal_repository=temporal_store, session_maker=cast(async_sessionmaker[AsyncSession], object()), ) @@ -256,7 +299,7 @@ async def fake_scoped_session( ) assert not generation_is_current - assert store.events == ["begin", "observations", "sections", "upsert"] + assert store.events == ["begin", "observations", "temporal", "sections", "upsert"] assert [call[0] for call in store.calls] == ["begin", "upsert"] assert transaction_count == 4 @@ -285,10 +328,12 @@ async def fake_scoped_session( store = RecordingRelationGenerationStore(begin_is_current=False) observation_store = RecordingObservationGenerationStore(events=store.events) section_store = RecordingSectionGenerationStore(events=store.events) + temporal_store = RecordingTemporalGenerationStore(events=store.events) publisher = RelationGenerationPublisher( relation_repository=store, observation_repository=observation_store, section_repository=section_store, + temporal_repository=temporal_store, session_maker=cast(async_sessionmaker[AsyncSession], object()), ) @@ -330,10 +375,12 @@ async def fake_scoped_session( events=store.events, ) section_store = RecordingSectionGenerationStore(events=store.events) + temporal_store = RecordingTemporalGenerationStore(events=store.events) publisher = RelationGenerationPublisher( relation_repository=store, observation_repository=observation_store, section_repository=section_store, + temporal_repository=temporal_store, session_maker=cast(async_sessionmaker[AsyncSession], object()), ) @@ -373,6 +420,7 @@ async def fake_scoped_session( ) store = RecordingRelationGenerationStore() observation_store = RecordingObservationGenerationStore(events=store.events) + temporal_store = RecordingTemporalGenerationStore(events=store.events) section_store = RecordingSectionGenerationStore( generation_is_current=False, events=store.events, @@ -381,6 +429,7 @@ async def fake_scoped_session( relation_repository=store, observation_repository=observation_store, section_repository=section_store, + temporal_repository=temporal_store, session_maker=cast(async_sessionmaker[AsyncSession], object()), ) @@ -390,7 +439,7 @@ async def fake_scoped_session( relations=[IndexedRelation("links_to", "Target", None)], sections=[], ) - assert store.events == ["begin", "observations", "sections"] + assert store.events == ["begin", "observations", "temporal", "sections"] assert [call[0] for call in store.calls] == ["begin"] assert section_store.calls == [(6, ())] assert transaction_count == 3 @@ -417,10 +466,12 @@ async def fake_scoped_session( store = RecordingRelationGenerationStore() observation_store = RecordingObservationGenerationStore(events=store.events) section_store = RecordingSectionGenerationStore(events=store.events) + temporal_store = RecordingTemporalGenerationStore(events=store.events) publisher = RelationGenerationPublisher( relation_repository=store, observation_repository=observation_store, section_repository=section_store, + temporal_repository=temporal_store, session_maker=cast(async_sessionmaker[AsyncSession], object()), ) @@ -457,10 +508,12 @@ async def test_relation_generation_publisher_rejects_non_self_pre_resolved_targe store = RecordingRelationGenerationStore() observation_store = RecordingObservationGenerationStore(events=store.events) section_store = RecordingSectionGenerationStore(events=store.events) + temporal_store = RecordingTemporalGenerationStore(events=store.events) publisher = RelationGenerationPublisher( relation_repository=store, observation_repository=observation_store, section_repository=section_store, + temporal_repository=temporal_store, session_maker=cast(async_sessionmaker[AsyncSession], object()), ) @@ -545,6 +598,7 @@ async def cleanup_relation_generations( relation_repository=_FailAfterPublicationBegins(), observation_repository=observation_repository, section_repository=NoteSectionRepository(project_id=sample_entity.project_id), + temporal_repository=MemoryTimeIndexRepository(project_id=sample_entity.project_id), session_maker=session_maker, ) with pytest.raises(OSError, match="relation chunk write failed"): @@ -574,6 +628,7 @@ async def cleanup_relation_generations( relation_repository=relation_repository, observation_repository=observation_repository, section_repository=NoteSectionRepository(project_id=sample_entity.project_id), + temporal_repository=MemoryTimeIndexRepository(project_id=sample_entity.project_id), session_maker=session_maker, ) assert await retry_publisher.publish( @@ -642,6 +697,7 @@ async def test_generation_zero_relation_forces_generation_publication( relation_repository=relation_repository, observation_repository=observation_repository, section_repository=NoteSectionRepository(project_id=sample_entity.project_id), + temporal_repository=MemoryTimeIndexRepository(project_id=sample_entity.project_id), session_maker=session_maker, ) assert await publisher.publish( diff --git a/tests/indexing/test_relation_persistence_temporal.py b/tests/indexing/test_relation_persistence_temporal.py new file mode 100644 index 000000000..5590d54c2 --- /dev/null +++ b/tests/indexing/test_relation_persistence_temporal.py @@ -0,0 +1,443 @@ +"""Publishing the valid-time projection under the note_content generation fence. + +Valid time is derived state: the markdown is the claim, these rows are its queryable +shadow, and every (re)index rebuilds them. Two properties keep that safe without adding +locks: + +* A stale writer no-ops. It never deletes the current rows and never inserts its own. +* Observations and their valid time move together. The projection addresses observation + rows by the ids the observation insert mints, so the two writes share one transaction + under one held fence -- the narrow exception the publisher documents. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass, field + +import pytest +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from basic_memory import db +from basic_memory.indexing.models import IndexedObservation +from basic_memory.indexing.relation_persistence import RelationGenerationPublisher +from basic_memory.models import Entity, NoteContent +from basic_memory.repository.memory_time_index_repository import ( + AcceptedTemporalAssertion, + MemoryTimeIndexRepository, + TemporalGenerationWriteResult, +) +from basic_memory.repository.note_section_repository import NoteSectionRepository +from basic_memory.repository.observation_repository import ( + AcceptedObservationWrite, + ObservationGenerationWriteResult, + ObservationRepository, +) +from basic_memory.repository.relation_repository import RelationRepository +from basic_memory.schemas.search import SearchItemType +from basic_memory.temporal import ( + TemporalAssertion, + TemporalRangeAxis, + TimeKind, + parse_range_literal, +) + +DATE = TemporalRangeAxis.DATE + + +def _assertion(literal: str, kind: TimeKind = TimeKind.EFFECTIVE) -> TemporalAssertion: + return TemporalAssertion( + time_kind=kind, + valid_during=parse_range_literal(literal, axis=DATE), + source_text=f"@{kind.value}{literal}", + ) + + +def _accepted( + source_id: int, literal: str, kind: TimeKind = TimeKind.EFFECTIVE +) -> AcceptedTemporalAssertion: + return AcceptedTemporalAssertion( + source_type=SearchItemType.OBSERVATION.value, + source_id=source_id, + assertion=_assertion(literal, kind), + ) + + +async def _add_note_content_generation( + session_maker: async_sessionmaker[AsyncSession], + entity: Entity, + *, + generation: int, +) -> None: + """Give the entity a note_content row at `generation`, which is the fence.""" + async with db.scoped_session(session_maker) as session: + session.add( + NoteContent( + entity_id=entity.id, + project_id=entity.project_id, + external_id=f"content-{entity.external_id}", + file_path=entity.file_path, + markdown_content="# Current\n", + db_version=generation, + db_checksum=f"checksum-{generation}", + file_write_status="synced", + ) + ) + + +# --- Repository-level fence behavior --- + + +@pytest.mark.asyncio +async def test_temporal_projection_replaced_under_the_generation_fence( + sample_entity: Entity, + session_maker: async_sessionmaker[AsyncSession], +): + """A later replace under the same fence discards every prior assertion.""" + repository = MemoryTimeIndexRepository(project_id=sample_entity.project_id) + await _add_note_content_generation(session_maker, sample_entity, generation=3) + + async with db.scoped_session(session_maker) as session: + await repository.replace_assertions_for_generation( + session, + entity_id=sample_entity.id, + generation=3, + assertions=[_accepted(1, "[2026-06-10,2026-07-27)")], + ) + async with db.scoped_session(session_maker) as session: + result = await repository.replace_assertions_for_generation( + session, + entity_id=sample_entity.id, + generation=3, + assertions=[ + _accepted(2, "[2026-07-27,)"), + _accepted(3, "[2026-01-01,2026-06-10)", TimeKind.DUE), + ], + ) + + assert result.generation_is_current + async with db.scoped_session(session_maker) as session: + rows = await repository.find_by_entity(session, sample_entity.id) + + assert [(row.source_id, row.time_kind) for row in rows] == [(2, "effective"), (3, "due")] + assert rows[0].lower_value == "2026-07-27" + assert rows[0].upper_value is None + + +@pytest.mark.asyncio +async def test_stale_generation_leaves_temporal_rows_untouched( + sample_entity: Entity, + session_maker: async_sessionmaker[AsyncSession], +): + """A stale writer no-ops instead of blocking: the current rows survive intact. + + This is the whole reason the fence exists rather than a lock. The loser of the race + writes nothing and reports it, and the winner's projection stands. + """ + repository = MemoryTimeIndexRepository(project_id=sample_entity.project_id) + await _add_note_content_generation(session_maker, sample_entity, generation=8) + async with db.scoped_session(session_maker) as session: + await repository.replace_assertions_for_generation( + session, + entity_id=sample_entity.id, + generation=8, + assertions=[_accepted(1, "[2026-07-27,)")], + ) + + async with db.scoped_session(session_maker) as session: + result = await repository.replace_assertions_for_generation( + session, + entity_id=sample_entity.id, + generation=7, + assertions=[_accepted(2, "[2026-01-01,2026-02-01)")], + ) + + assert not result.generation_is_current + async with db.scoped_session(session_maker) as session: + rows = await repository.find_by_entity(session, sample_entity.id) + assert [(row.source_id, row.lower_value) for row in rows] == [(1, "2026-07-27")] + + +@pytest.mark.asyncio +async def test_empty_assertion_set_wipes_prior_rows( + sample_entity: Entity, + session_maker: async_sessionmaker[AsyncSession], +): + """Removing every qualifier from a note must remove its valid time, not keep it.""" + repository = MemoryTimeIndexRepository(project_id=sample_entity.project_id) + await _add_note_content_generation(session_maker, sample_entity, generation=5) + async with db.scoped_session(session_maker) as session: + await repository.replace_assertions_for_generation( + session, + entity_id=sample_entity.id, + generation=5, + assertions=[_accepted(1, "[2026-06-10,2026-07-27)")], + ) + + async with db.scoped_session(session_maker) as session: + result = await repository.replace_assertions_for_generation( + session, + entity_id=sample_entity.id, + generation=5, + assertions=[], + ) + + assert result.generation_is_current + async with db.scoped_session(session_maker) as session: + assert await repository.find_by_entity(session, sample_entity.id) == [] + + +@pytest.mark.asyncio +async def test_find_for_sources_returns_nothing_for_an_empty_request( + sample_entity: Entity, + session_maker: async_sessionmaker[AsyncSession], +): + """Hydrating an empty result page must not issue a query at all.""" + repository = MemoryTimeIndexRepository(project_id=sample_entity.project_id) + + async with db.scoped_session(session_maker) as session: + assert await repository.find_for_sources(session, []) == [] + + +# --- Publisher-level: observations and their valid time move together --- + + +@pytest.mark.asyncio +async def test_publisher_addresses_the_observation_rows_it_just_minted( + sample_entity: Entity, + observation_repository: ObservationRepository, + relation_repository: RelationRepository, + session_maker: async_sessionmaker[AsyncSession], +): + """Each assertion lands on the id of the observation that carried the qualifier. + + Observation rows are wiped and recreated on every publication, so their ids only + exist after the insert. Pairing by document order is what connects a qualifier back + to its own statement rather than to its neighbour. + """ + temporal_repository = MemoryTimeIndexRepository(project_id=sample_entity.project_id) + publisher = RelationGenerationPublisher( + relation_repository=relation_repository, + observation_repository=observation_repository, + section_repository=NoteSectionRepository(project_id=sample_entity.project_id), + temporal_repository=temporal_repository, + session_maker=session_maker, + ) + await _add_note_content_generation(session_maker, sample_entity, generation=1) + + published = await publisher.publish( + entity_id=sample_entity.id, + generation=1, + relations=[], + observations=[ + IndexedObservation( + content="The cache layer will use Redis.", + category="decision", + context=None, + tags=None, + temporal=(_assertion("[2026-06-10,2026-07-27)"),), + ), + IndexedObservation( + content="The cache layer will use Memcached.", + category="decision", + context=None, + tags=None, + temporal=(_assertion("[2026-07-27,)"),), + ), + IndexedObservation( + content="The queue layer will use RabbitMQ.", + category="decision", + context=None, + tags=None, + ), + ], + ) + + assert published + async with db.scoped_session(session_maker) as session: + observations = await observation_repository.find_by_entity(session, sample_entity.id) + rows = await temporal_repository.find_by_entity(session, sample_entity.id) + + ids_by_content = {observation.content: observation.id for observation in observations} + assert {row.source_id: row.source_text for row in rows} == { + ids_by_content["The cache layer will use Redis."]: "@effective[2026-06-10,2026-07-27)", + ids_by_content["The cache layer will use Memcached."]: "@effective[2026-07-27,)", + } + # The undated observation contributes no row: it makes no claim. + assert ids_by_content["The queue layer will use RabbitMQ."] not in { + row.source_id for row in rows + } + assert all(row.source_type == SearchItemType.OBSERVATION.value for row in rows) + + +@pytest.mark.asyncio +async def test_republishing_rebuilds_the_projection_against_the_new_row_ids( + sample_entity: Entity, + observation_repository: ObservationRepository, + relation_repository: RelationRepository, + session_maker: async_sessionmaker[AsyncSession], +): + """Re-indexing re-mints observation ids, and the projection follows them. + + This is the invariant that makes the shared transaction necessary: if the temporal + write ran later, it would address ids the observation wipe had already discarded. + """ + temporal_repository = MemoryTimeIndexRepository(project_id=sample_entity.project_id) + publisher = RelationGenerationPublisher( + relation_repository=relation_repository, + observation_repository=observation_repository, + section_repository=NoteSectionRepository(project_id=sample_entity.project_id), + temporal_repository=temporal_repository, + session_maker=session_maker, + ) + await _add_note_content_generation(session_maker, sample_entity, generation=1) + observation = IndexedObservation( + content="The cache layer will use Redis.", + category="decision", + context=None, + tags=None, + temporal=(_assertion("[2026-06-10,2026-07-27)"),), + ) + + assert await publisher.publish( + entity_id=sample_entity.id, generation=1, relations=[], observations=[observation] + ) + assert await publisher.publish( + entity_id=sample_entity.id, generation=1, relations=[], observations=[observation] + ) + + async with db.scoped_session(session_maker) as session: + observations = await observation_repository.find_by_entity(session, sample_entity.id) + rows = await temporal_repository.find_by_entity(session, sample_entity.id) + + assert [row.source_id for row in rows] == [observation.id for observation in observations] + + +@dataclass(slots=True) +class _MisalignedObservationStore: + """An observation store that returns the wrong number of row ids.""" + + calls: list[int] = field(default_factory=list) + + async def replace_observations_for_generation( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + observations: Sequence[AcceptedObservationWrite], + ) -> ObservationGenerationWriteResult: + del session, entity_id, generation + self.calls.append(len(observations)) + return ObservationGenerationWriteResult(generation_is_current=True, observation_ids=(1,)) + + +@dataclass(slots=True) +class _UnreachableTemporalStore: + """A temporal store that must never be called.""" + + async def replace_assertions_for_generation( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + assertions: Sequence[AcceptedTemporalAssertion], + ) -> TemporalGenerationWriteResult: # pragma: no cover - reaching this is the failure + raise AssertionError("misaligned observation ids must be caught before publication") + + +@pytest.mark.asyncio +async def test_misaligned_observation_ids_fail_loudly( + sample_entity: Entity, + relation_repository: RelationRepository, + session_maker: async_sessionmaker[AsyncSession], +): + """Pairing by position is only safe while the two sequences agree in length. + + A mismatch would silently attach one statement's valid time to another, which is a + wrong answer rather than a stale one -- so it raises instead of publishing. + """ + publisher = RelationGenerationPublisher( + relation_repository=relation_repository, + observation_repository=_MisalignedObservationStore(), + section_repository=NoteSectionRepository(project_id=sample_entity.project_id), + temporal_repository=_UnreachableTemporalStore(), + session_maker=session_maker, + ) + await _add_note_content_generation(session_maker, sample_entity, generation=1) + + with pytest.raises(ValueError, match="returned 1 row ids for 2 observations"): + await publisher.publish( + entity_id=sample_entity.id, + generation=1, + relations=[], + observations=[ + IndexedObservation("First", "decision", None, None), + IndexedObservation("Second", "decision", None, None), + ], + ) + + +@pytest.mark.asyncio +async def test_observation_write_returns_the_ids_it_minted( + sample_entity: Entity, + observation_repository: ObservationRepository, + session_maker: async_sessionmaker[AsyncSession], +): + """The observation replace reports its new row ids in document order.""" + await _add_note_content_generation(session_maker, sample_entity, generation=2) + + async with db.scoped_session(session_maker) as session: + result = await observation_repository.replace_observations_for_generation( + session, + entity_id=sample_entity.id, + generation=2, + observations=[ + AcceptedObservationWrite("First", "decision", None, None), + AcceptedObservationWrite("Second", "decision", None, None), + ], + ) + + assert result.generation_is_current + async with db.scoped_session(session_maker) as session: + observations = await observation_repository.find_by_entity(session, sample_entity.id) + assert result.observation_ids == tuple(observation.id for observation in observations) + + +@pytest.mark.asyncio +async def test_stale_observation_fence_publishes_no_valid_time( + sample_entity: Entity, + observation_repository: ObservationRepository, + relation_repository: RelationRepository, + session_maker: async_sessionmaker[AsyncSession], +): + """A publication that lost its fence leaves both projections as they were.""" + temporal_repository = MemoryTimeIndexRepository(project_id=sample_entity.project_id) + publisher = RelationGenerationPublisher( + relation_repository=relation_repository, + observation_repository=observation_repository, + section_repository=NoteSectionRepository(project_id=sample_entity.project_id), + temporal_repository=temporal_repository, + session_maker=session_maker, + ) + await _add_note_content_generation(session_maker, sample_entity, generation=9) + + published = await publisher.publish( + entity_id=sample_entity.id, + generation=4, + relations=[], + observations=[ + IndexedObservation( + content="The cache layer will use Redis.", + category="decision", + context=None, + tags=None, + temporal=(_assertion("[2026-06-10,2026-07-27)"),), + ) + ], + ) + + assert not published + async with db.scoped_session(session_maker) as session: + assert await temporal_repository.find_by_entity(session, sample_entity.id) == [] diff --git a/tests/markdown/test_entity_parser.py b/tests/markdown/test_entity_parser.py index df55066fa..b37632978 100644 --- a/tests/markdown/test_entity_parser.py +++ b/tests/markdown/test_entity_parser.py @@ -3,8 +3,10 @@ from datetime import UTC, datetime from pathlib import Path from textwrap import dedent +from typing import Any import pytest +from loguru import logger from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter, Relation from basic_memory.markdown.entity_parser import parse @@ -428,6 +430,41 @@ async def test_graph_silent_note_still_gets_sections(entity_parser): assert [section.heading for section in entity.sections] == ["Extracted"] +@pytest.mark.asyncio +async def test_malformed_qualifier_logs_diagnostic_with_file_path(entity_parser): + """A refused temporal qualifier warns with the path, so the author can fix the line. + + The typed `temporal_error` field carries the same message to programmatic callers; + this layer is the only one that knows which file the observation came from. + + The path is asserted in its forward-slash form on every platform. That is not a + convenience for the test: it is the same spelling `entity.file_path`, permalinks, + and search rows use, so the name in the warning is one the author can search for. + A `WindowsPath` interpolated directly would print `decisions\\cache-layer.md`. + """ + records: list[Any] = [] + sink_id = logger.add(lambda message: records.append(message.record), level="WARNING") + try: + entity = await entity_parser.parse_markdown_content( + Path("decisions/cache-layer.md"), + "# Cache\n- [decision] @asserted[2026-06-10,) The cache layer will use Redis.\n", + ) + finally: + logger.remove(sink_id) + + [observation] = entity.observations + assert observation.temporal == [] + assert "unknown temporal kind 'asserted'" in (observation.temporal_error or "") + # The qualifier is never dropped from the content, only from the projection. + assert observation.content.startswith("@asserted[2026-06-10,)") + + messages = [record["message"] for record in records] + assert any( + "decisions/cache-layer.md" in message and "Temporal qualifier ignored" in message + for message in messages + ), messages + + # @pytest.mark.asyncio # async def test_parse_file_invalid_yaml(test_config, entity_parser): # """Test parsing file with invalid YAML frontmatter.""" diff --git a/tests/markdown/test_temporal_qualifier.py b/tests/markdown/test_temporal_qualifier.py new file mode 100644 index 000000000..724cb11d6 --- /dev/null +++ b/tests/markdown/test_temporal_qualifier.py @@ -0,0 +1,936 @@ +"""Parsing and round-tripping SPEC-82 temporal qualifiers on observations. + +Three rules shape every test here: + +* **One grammar.** `@[kind]` for a precise interval, `@[kind:]` + for an unquoted point, and `@[kind:]""` for a quoted one. The kind is optional + in all three; an unquoted point that omits it must begin with a digit. +* **Silent when it is not time.** If the payload does not read as a date, the token is + ordinary content and nothing is reported. Prose is full of `@`, and diagnosing every + one of them would be noise. +* **Diagnostics only where the author plainly meant a qualifier.** An unknown kind, an + unterminated quote, and an unquoted date the one-token rule truncated are each + reported, because each one has a fix the message can name. + +And in every case a qualifier that was not accepted is **never dropped**: its text +stays in the observation content, so the line indexes and round-trips exactly as it did +before valid time existed. +""" + +from datetime import datetime, timedelta + +import pytest + +from basic_memory import config as config_module +from basic_memory.config import ConfigManager +from basic_memory.markdown.entity_parser import parse +from basic_memory.markdown.schemas import Observation +from basic_memory.markdown.temporal_qualifier import parse_temporal_qualifier +from basic_memory.temporal import DateOrder, TemporalRangeAxis, TimeKind + + +@pytest.fixture(autouse=True) +def isolated_config(config_home, monkeypatch): + """Point config resolution at a temp HOME for the whole module. + + The point form consults `date_order`, so parsing a qualifier reads configuration. + `config_home` patches HOME; resetting the process cache keeps one test's config + from leaking into the next. + """ + monkeypatch.setattr(config_module, "_CONFIG_CACHE", None) + monkeypatch.setattr(config_module, "_CONFIG_MTIME", None) + monkeypatch.setattr(config_module, "_CONFIG_SIZE", None) + return config_home + + +def _observation(line: str) -> Observation: + """Parse a single observation line through the real markdown pipeline.""" + [observation] = parse(line).observations + return observation + + +def _refusal(line: str) -> Observation: + """Parse a line whose qualifier must be refused, and assert the shared contract.""" + observation = _observation(line) + assert observation.temporal == [] + assert observation.temporal_error is not None + return observation + + +# --- Acceptance 1: undated notes are untouched --- + + +def test_observation_without_qualifier_parses_byte_identically(): + """A note that asserts no valid time behaves exactly as it did before SPEC-82.""" + observation = _observation("- [decision] The cache layer will use Redis. #infra (agreed)") + + assert observation.category == "decision" + assert observation.content == "The cache layer will use Redis. #infra" + assert observation.tags == ["infra"] + assert observation.context == "agreed" + assert observation.temporal == [] + assert observation.temporal_error is None + assert str(observation) == "- [decision] The cache layer will use Redis. #infra (agreed)" + + +# --- Acceptance 2: round trip preserves kind and bounds --- + + +@pytest.mark.parametrize( + "qualifier", + [ + # The range literal: the precise form, unchanged by the point form's arrival. + "@effective[2026-06-10,2026-07-27)", + "@effective(2026-06-10,2026-07-27]", + "@effective[2026-06-10,2026-07-27]", + "@effective(2026-06-10,2026-07-27)", + "@effective[2026-06-10,)", + "@effective(,2026-07-27)", + "@valid[2026-01-01,2026-12-31)", + "@occurred[2026-07-27T18:42:00Z,2026-07-27T19:00:00Z)", + "@due[2026-07-27T18:42:00+02:00,)", + "@mentioned[2026-07-27T18:42:00.123456Z,2026-07-28T00:00:00Z)", + "@[2026-06-10,2026-07-27)", + # The point: the convenient form, with and without a kind. + "@effective:2026-07-27", + "@occurred:2026-07-27T18:42:00Z", + "@due:2026-07", + "@2026-07-27", + "@2026-07", + "@2026", + "@10/07/2026", + ], +) +def test_qualifier_round_trips_verbatim(qualifier: str): + """Serializing a parsed observation replays the author's exact qualifier text. + + `valid_during` holds normalized bounds -- UTC, microsecond precision, a canonical + interval -- but the author's own spelling is what gets written back, so a + parse/serialize cycle never rewrites their file. + """ + line = f"- [decision] {qualifier} The cache layer will use Redis." + + observation = _observation(line) + + [assertion] = observation.temporal + assert observation.temporal_error is None + assert observation.content == "The cache layer will use Redis." + assert assertion.source_text == qualifier + assert assertion.extractor == "observation" + assert str(observation) == line + + +def test_qualifier_carries_its_kind_and_bounds(): + """The parsed assertion is the interval the author wrote, of the kind they named.""" + observation = _observation( + "- [decision] @effective[2026-06-10,2026-07-27) The cache layer will use Redis." + ) + + [assertion] = observation.temporal + assert assertion.time_kind is TimeKind.EFFECTIVE + assert assertion.valid_during.axis is TemporalRangeAxis.DATE + assert assertion.valid_during.lower == "2026-06-10" + assert assertion.valid_during.upper == "2026-07-27" + assert assertion.valid_during.lower_inclusive is True + assert assertion.valid_during.upper_inclusive is False + assert str(assertion.valid_during) == "[2026-06-10,2026-07-27)" + + +def test_a_closed_qualifier_is_stored_half_open_without_rewriting_the_line(): + """The two forms coexist: canonical bounds for the index, the author's text on disk. + + `[2026-06-10,2026-07-27]` means "through July 27", which the discrete canonical form + spells `[2026-06-10,2026-07-28)`. That normalization is the projection's business -- + `source_text` keeps the author's words, so serializing the note writes the file back + exactly as they wrote it. + """ + line = "- [decision] @effective[2026-06-10,2026-07-27] The cache layer will use Redis." + + observation = _observation(line) + + [assertion] = observation.temporal + assert assertion.source_text == "@effective[2026-06-10,2026-07-27]" + assert str(assertion.valid_during) == "[2026-06-10,2026-07-28)" + assert assertion.valid_during.upper_inclusive is False + assert str(observation) == line + + +def test_qualifier_is_peeled_before_context_and_tags(): + """Peel order matters: the context rule would otherwise steal a `)` qualifier. + + An exclusive-upper qualifier ends in `)`, and the context rule is a bare + suffix match, so parsing context first would claim the qualifier and leave the + observation content empty -- which the plugin then drops outright. + """ + observation = _observation( + "- [decision] @effective(2026-06-10,2026-07-27] Use Redis #infra (agreed)" + ) + + [assertion] = observation.temporal + assert assertion.source_text == "@effective(2026-06-10,2026-07-27]" + assert observation.content == "Use Redis #infra" + assert observation.context == "agreed" + # Qualifier digits are not tags: the peel happens before the tag scan. + assert observation.tags == ["infra"] + + +def test_qualifier_alone_on_the_line_still_parses(): + """A qualifier with no trailing context is the common case, not an edge case.""" + observation = _observation("- [decision] @effective(2026-06-10,2026-07-27] Use Redis") + + [assertion] = observation.temporal + assert assertion.source_text == "@effective(2026-06-10,2026-07-27]" + assert observation.content == "Use Redis" + assert observation.context is None + + +# --- The point form: what each precision means --- + + +@pytest.mark.parametrize( + ("qualifier", "literal", "axis"), + [ + # A year and a month are periods the author delimited by writing them. + ("@2026", "[2026-01-01,2027-01-01)", TemporalRangeAxis.DATE), + ("@2026-06", "[2026-06-01,2026-07-01)", TemporalRangeAxis.DATE), + # A date says when something started and leaves it open. + ("@2026-06-10", "[2026-06-10,)", TemporalRangeAxis.DATE), + # So does a moment, on the instant axis. + ( + "@2026-06-10T14:00:00", + "[2026-06-10T14:00:00.000000Z,)", + TemporalRangeAxis.INSTANT, + ), + ( + "@2026-06-10T14:00:00Z", + "[2026-06-10T14:00:00.000000Z,)", + TemporalRangeAxis.INSTANT, + ), + ( + "@2026-06-10T14:00:00+02:00", + "[2026-06-10T12:00:00.000000Z,)", + TemporalRangeAxis.INSTANT, + ), + ], +) +def test_point_qualifier_canonicalizes_to_the_span_its_precision_covers( + qualifier: str, literal: str, axis: TemporalRangeAxis +): + observation = _observation(f"- [decision] {qualifier} The cutover ran.") + + [assertion] = observation.temporal + assert str(assertion.valid_during) == literal + assert assertion.valid_during.axis is axis + + +def test_a_point_with_no_kind_is_filed_as_valid_time(): + """`@2026-06-10` says when the statement holds, without narrowing how.""" + observation = _observation("- [decision] @2026-06-10 The cache layer will use Redis.") + + [assertion] = observation.temporal + assert assertion.time_kind is TimeKind.VALID + + +def test_a_range_literal_with_no_kind_is_filed_as_valid_time(): + """The kind is optional in both forms, and defaults the same way in both.""" + observation = _observation("- [decision] @[2026-06-10,2026-07-27) Use Redis.") + + [assertion] = observation.temporal + assert assertion.time_kind is TimeKind.VALID + assert str(assertion.valid_during) == "[2026-06-10,2026-07-27)" + + +@pytest.mark.parametrize( + ("qualifier", "kind"), + [ + ("@effective:2026-06-10", TimeKind.EFFECTIVE), + ("@occurred:2026-06-10", TimeKind.OCCURRED), + ("@due:2026-06-10", TimeKind.DUE), + ("@mentioned:2026-06-10", TimeKind.MENTIONED), + ("@valid:2026-06-10", TimeKind.VALID), + ], +) +def test_point_qualifier_names_its_kind_with_a_colon(qualifier: str, kind: TimeKind): + """`:` separates kind from date; a date can start with a letter, so it is needed.""" + observation = _observation(f"- [decision] {qualifier} The cutover ran.") + + [assertion] = observation.temporal + assert assertion.time_kind is kind + assert str(assertion.valid_during) == "[2026-06-10,)" + + +def test_a_point_with_a_kind_accepts_a_relative_date(): + """With a kind the author has said what they mean, so any readable date is taken. + + Relative wording resolves at parse time and is re-resolved on every index pass. + That is documented behavior, not a mistake to warn about. + """ + observation = _observation("- [decision] @occurred:yesterday The cutover ran.") + + [assertion] = observation.temporal + yesterday = datetime.now().date() - timedelta(days=1) + assert assertion.valid_during.lower == yesterday.isoformat() + assert observation.content == "The cutover ran." + + +@pytest.mark.parametrize( + "qualifier", + [ + # Words: dateparser reads several of these as months or years. + "@yesterday", + "@may", + "@v2", + "@june", + # Too short to be a year: list markers and version numbers, which dateparser + # would otherwise read as January, 2012, and March 5. + "@1", + "@12", + "@3.5", + "@5-3", + ], +) +def test_a_point_with_no_kind_must_be_digit_led_and_year_wide(qualifier: str): + """A bare `@token` that short is a mention, a version, or a list marker. + + Accepting what dateparser makes of these would silently file wrong valid time on + ordinary prose. An author who really means one writes the kind: `@occurred:may`. + """ + observation = _observation(f"- [decision] {qualifier} shipped the cutover.") + + assert observation.temporal == [] + assert observation.temporal_error is None + assert observation.content.startswith(qualifier) + + +def test_a_word_point_is_read_only_when_it_names_a_specific_day(): + """A kind opens the form to words, but not to words that name only a period. + + `yesterday` resolves to one day and is taken. `may` resolves to a whole month, and + a bare month name at the head of a line is either prose or -- worse -- the first + token of `May 10, 2026`, where reading it would file May 2026 and leave `10, 2026` + behind as content. + """ + day = _observation("- [decision] @occurred:yesterday The cutover ran.") + [assertion] = day.temporal + assert assertion.time_kind is TimeKind.OCCURRED + assert day.content == "The cutover ran." + + period = _observation("- [decision] @occurred:may The cutover ran.") + assert period.temporal == [] + assert period.temporal_error is None + assert period.content.startswith("@occurred:may") + + +# --- The flexible vocabulary, as the qualifier grammar sees it --- +# +# `parse_authored_point` reads far more spellings than these (tests/test_temporal.py +# pins that vocabulary). The grammar is narrower on purpose, and this section is the +# boundary between the two: a qualifier is one whitespace-delimited token, because +# dateparser also reads `June 10, 2026 The` and `2026-06-10 The`, so there is no way to +# tell where a multi-word date stops without swallowing the author's prose. + + +@pytest.mark.parametrize( + ("qualifier", "literal", "axis"), + [ + # Single-token absolute dates, with a kind and without. + ("@occurred:2026-06-10", "[2026-06-10,)", TemporalRangeAxis.DATE), + ("@occurred:03/04/2026", "[2026-04-03,)", TemporalRangeAxis.DATE), + ( + "@occurred:2026-06-10T10:00:00", + "[2026-06-10T10:00:00.000000Z,)", + TemporalRangeAxis.INSTANT, + ), + # A kind admits a word, as long as it names one day. + ("@occurred:today", None, TemporalRangeAxis.DATE), + ("@occurred:yesterday", None, TemporalRangeAxis.DATE), + ], +) +def test_single_token_points_are_accepted(qualifier: str, literal: str | None, axis): + observation = _observation(f"- [decision] {qualifier} The cutover ran.") + + [assertion] = observation.temporal + assert observation.content == "The cutover ran." + assert assertion.valid_during.axis is axis + if literal is not None: + assert str(assertion.valid_during) == literal + + +@pytest.mark.parametrize( + ("qualifier", "reported"), + [ + # Multi-word dates: only the first token reaches the reader, and each of these + # first tokens is refused, so the whole line stays content rather than being + # half-read. `@occurred:"June 10, 2026"` says it in one delimited token. + ("@occurred:June 10, 2026", True), + ("@occurred:Jan 15, 2024", True), + ("@occurred:10 June 2026", False), + ("@occurred:2 days ago", False), + ("@occurred:last week", False), + ], +) +def test_multi_word_dates_stay_content_whole(qualifier: str, reported: bool): + """The reader understands these; the unquoted grammar cannot delimit them. + + What matters is that an undelimitable date is left *entirely* alone: no coarse + assertion filed from its first token, and no words eaten out of the content. Whether + the author additionally *hears* about it is the digit-follows signal's business, + pinned below -- the line itself is untouched either way. + """ + line = f"- [decision] {qualifier} The cutover ran." + + observation = _observation(line) + + assert observation.temporal == [] + assert observation.content == f"{qualifier} The cutover ran." + assert str(observation) == line + assert (observation.temporal_error is not None) is reported + + +def test_a_multi_word_date_is_read_up_to_its_first_token_when_that_token_stands_alone(): + """The one partial read the token rule allows, pinned so it is a known boundary. + + `2026-06-10` is a complete date by itself, so the qualifier claims it and the clock + reading stays in the content. The assertion is coarser than the author meant -- a + date, not an instant -- but it is not wrong, and nothing is lost from the line. + """ + observation = _observation("- [decision] @occurred:2026-06-10 10:00 AM The cutover ran.") + + [assertion] = observation.temporal + assert str(assertion.valid_during) == "[2026-06-10,)" + assert assertion.valid_during.axis is TemporalRangeAxis.DATE + assert observation.content == "10:00 AM The cutover ran." + + +# --- The quoted point: a date the author delimited --- +# +# Quotes are how a multi-word date is written. They move the token boundary from the +# next space to the closing quote, which is the whole reason the one-token guards do not +# apply inside them: the author said where the date ends, so nothing can be truncated. + + +@pytest.mark.parametrize( + ("qualifier", "literal", "kind"), + [ + ('@occurred:"June 10, 2026"', "[2026-06-10,)", TimeKind.OCCURRED), + ('@effective:"10 June 2026"', "[2026-06-10,)", TimeKind.EFFECTIVE), + # Month-only and year-only: coarse on purpose, and delimited, so they are read. + ('@occurred:"June 2026"', "[2026-06-01,2026-07-01)", TimeKind.OCCURRED), + # With no kind, exactly like the bare point form -- filed as valid time. + ('@"June 10, 2026"', "[2026-06-10,)", TimeKind.VALID), + ], +) +def test_quoted_point_reads_a_multi_word_date(qualifier: str, literal: str, kind: TimeKind): + """The quoted form's payload goes to the date reader whole, spaces and all.""" + line = f"- [decision] {qualifier} The cutover ran." + + observation = _observation(line) + + [assertion] = observation.temporal + assert observation.temporal_error is None + assert assertion.time_kind is kind + assert str(assertion.valid_during) == literal + assert assertion.valid_during.axis is TemporalRangeAxis.DATE + assert observation.content == "The cutover ran." + # Quotes are part of the qualifier, so they round-trip with it. + assert assertion.source_text == qualifier + assert str(observation) == line + + +@pytest.mark.parametrize( + "point", + [ + "2026-01-01T10:00:00.1234567", + "2026-01-01T10:00:00.1234567Z", + "2026-01-01T10:00:00.1234567+02:00", + "2026-01-01T10:00:00." + "1" * 30, + ], +) +def test_a_point_finer_than_a_microsecond_stays_content_in_both_forms(point: str): + """An over-precise instant is refused whichever form carries it to the reader. + + Both forms filed `[2026-01-01T10:00:00.123456Z,)` -- the authored instant with its + last digits dropped, and no sign to the author that anything was lost. The quoted form + reached it by a different route than the bare one: quoting suppresses the truncation + guards, on the reasoning that a delimited value cannot be a truncated *token*. That is + still true, and beside the point here -- the loss is inside the value, so only refusing + the point itself covers both. Pinned together so a fix to one form cannot miss the + other. + """ + for qualifier in (f"@occurred:{point}", f'@occurred:"{point}"'): + line = f"- [decision] {qualifier} The cutover ran." + + observation = _observation(line) + + assert observation.temporal == [] + # Refused, not reported: how someone spelled a date is not a diagnostic this + # feature issues. The line keeps every character and stays full-text searchable. + assert observation.temporal_error is None + assert observation.content == f"{qualifier} The cutover ran." + assert str(observation) == line + + +def test_a_point_at_exactly_microsecond_precision_is_still_filed(): + """The boundary the refusal above stops at, end to end through the parser.""" + for qualifier in ( + "@occurred:2026-01-01T10:00:00.123456", + '@occurred:"2026-01-01T10:00:00.123456"', + ): + observation = _observation(f"- [decision] {qualifier} The cutover ran.") + + [assertion] = observation.temporal + assert assertion.time_kind is TimeKind.OCCURRED + assert str(assertion.valid_during) == "[2026-01-01T10:00:00.123456Z,)" + assert assertion.valid_during.axis is TemporalRangeAxis.INSTANT + assert observation.content == "The cutover ran." + + +def test_a_quoted_relative_date_is_read_where_its_unquoted_form_is_not(): + """`2 days ago` always read fine; only the token rule kept it out.""" + quoted = _observation('- [decision] @occurred:"2 days ago" The cutover ran.') + + [assertion] = quoted.temporal + two_days_ago = datetime.now().date() - timedelta(days=2) + assert assertion.valid_during.lower == two_days_ago.isoformat() + assert quoted.content == "The cutover ran." + + unquoted = _observation("- [decision] @occurred:2 days ago The cutover ran.") + assert unquoted.temporal == [] + + +def test_a_quoted_month_is_filed_where_the_specific_day_guard_refuses_it(): + """The guard exists to catch truncation, and a delimited value cannot be truncated. + + Unquoted, `June` is refused because it may be the head of `June 2026`. Quoted, the + author has already said the date is exactly that month. + """ + quoted = _observation('- [decision] @occurred:"June 2026" The cutover ran.') + + [assertion] = quoted.temporal + assert str(assertion.valid_during) == "[2026-06-01,2026-07-01)" + assert quoted.content == "The cutover ran." + + unquoted = _observation("- [decision] @occurred:June 2026 The cutover ran.") + assert unquoted.temporal == [] + + +def test_a_quoted_clock_reading_is_read_whole_where_the_token_rule_truncates_it(): + """The one partial read the token rule allows, undone by delimiting the value. + + Unquoted, `@occurred:2026-06-10 10:00 AM` files a calendar date and leaves the clock + reading in the content (pinned above). Quoted, the same text files the instant the + author meant, and nothing is left behind. + """ + observation = _observation('- [decision] @occurred:"2026-06-10 10:00 AM" The cutover ran.') + + [assertion] = observation.temporal + assert str(assertion.valid_during) == "[2026-06-10T10:00:00.000000Z,)" + assert assertion.valid_during.axis is TemporalRangeAxis.INSTANT + assert observation.content == "The cutover ran." + + +def test_the_closing_quote_ends_the_token_and_the_rest_stays_content(): + """Content after the closing quote is ordinary content, quotes and digits included. + + Whitespace no longer delimits the token, so the peel has to stop at the quote and + hand back everything after it exactly as written -- including text that would have + been read as more date had the scan kept going. + """ + line = ( + '- [decision] @occurred:"June 10, 2026" She said "go", then 10, 2026 ' + "shipped #infra (agreed)" + ) + + observation = _observation(line) + + [assertion] = observation.temporal + assert assertion.source_text == '@occurred:"June 10, 2026"' + assert observation.content == 'She said "go", then 10, 2026 shipped #infra' + assert observation.tags == ["infra"] + assert observation.context == "agreed" + assert str(observation) == line + + +def test_content_may_follow_the_closing_quote_with_no_space(): + """The quote is the boundary, so nothing else has to mark it.""" + observation = _observation('- [decision] @occurred:"June 10, 2026"The cutover ran.') + + [assertion] = observation.temporal + assert assertion.source_text == '@occurred:"June 10, 2026"' + assert observation.content == "The cutover ran." + + +@pytest.mark.parametrize( + "qualifier", + ['@occurred:""', '@occurred:"not a date"', '@"the cutover week"'], +) +def test_a_quoted_payload_that_is_not_a_date_stays_content_silently(qualifier: str): + """Quoting says where the value ends, not that the value is a date.""" + observation = _observation(f"- [decision] {qualifier} The cutover ran.") + + assert observation.temporal == [] + assert observation.temporal_error is None + assert observation.content == f"{qualifier} The cutover ran." + + +def test_a_quoted_point_still_reports_an_unknown_kind(): + """The quoted form is a spelling of the point, so it keeps the point's diagnostic.""" + observation = _refusal('- [decision] @asserted:"June 10, 2026" The cutover ran.') + + assert "unknown temporal kind 'asserted'" in (observation.temporal_error or "") + assert observation.content.startswith('@asserted:"June 10, 2026"') + + +def test_an_unterminated_quote_is_reported_instead_of_swallowing_the_line(): + """Reading on would hand the author's prose to the date reader; refusing keeps it.""" + line = '- [decision] @occurred:"June 10, 2026 The cutover ran.' + + observation = _refusal(line) + + assert "unterminated quote" in (observation.temporal_error or "") + # The fix is shown, not described. + assert '@occurred:"June 10, 2026"' in (observation.temporal_error or "") + assert observation.content == '@occurred:"June 10, 2026 The cutover ran.' + assert str(observation) == line + + +def test_an_escaped_quote_belongs_to_the_value_and_cannot_close_it(): + r"""`\"` is part of the date text, which is why this line has no closing quote left.""" + observation = _refusal('- [decision] @occurred:"June 10, 2026\\" The cutover ran.') + + assert "unterminated quote" in (observation.temporal_error or "") + assert observation.content == '@occurred:"June 10, 2026\\" The cutover ran.' + + +# --- The truncation diagnostic: when the one-token rule costs a date --- + + +def test_a_truncated_date_names_the_quoted_form_as_the_fix(): + """`@occurred:June 10, 2026` is the shape quoting exists for, so say so once.""" + observation = _refusal("- [decision] @occurred:June 10, 2026 The cutover ran.") + + error = observation.temporal_error or "" + assert "'@occurred:June'" in error + assert "names only a month or a year" in error + assert '@occurred:"June 10, 2026"' in error + # Reported, never half-read: the line is still exactly what the author wrote. + assert observation.content == "@occurred:June 10, 2026 The cutover ran." + + +def test_a_too_short_number_followed_by_a_digit_names_the_quoted_form_too(): + """The other guard gets the same treatment, with its own reason and the same fix.""" + observation = _refusal("- [note] @12 2026 was the year of the cutover.") + + error = observation.temporal_error or "" + assert "'@12'" in error + assert "is narrower than a year" in error + # A point with no kind is fixed by the quoted form with no kind. + assert '@"June 10, 2026"' in error + + +@pytest.mark.parametrize( + "line", + [ + # Prose follows, so nothing suggests a date was cut short. + "- [decision] @occurred:June the cat sat on the mat", + "- [decision] @occurred:may The cutover ran.", + "- [decision] @1 shipped the cutover.", + # Nothing follows at all. + "- [decision] @occurred:June", + # A digit follows, but `@vol:` is an ordinary `@word:` marker, not a kind. + "- [note] @vol:2 3 pages of notes", + ], +) +def test_a_refused_point_stays_silent_when_the_line_did_not_continue_the_date(line: str): + """Today's behavior, kept: the diagnostic fires on one signal, not on every refusal.""" + observation = _observation(line) + + assert observation.temporal == [] + assert observation.temporal_error is None + + +@pytest.mark.parametrize( + ("date_order", "expected_lower"), + [("YMD", "2026-04-03"), ("DMY", "2026-04-03"), ("MDY", "2026-03-04")], +) +def test_a_slash_date_with_a_kind_follows_the_configured_order( + date_order: DateOrder, expected_lower: str +): + """`@occurred:03/04/2026` resolves by preference, through the real parse path.""" + observation = parse_temporal_qualifier( + "@occurred:03/04/2026 The cutover ran.", date_order=date_order + ) + + [assertion] = observation.assertions + assert assertion.valid_during.lower == expected_lower + assert observation.content == "The cutover ran." + + +# --- Date order comes from configuration --- + + +def test_configured_date_order_decides_an_ambiguous_slash_date(monkeypatch): + """`@10/07/2026` is July 10 by default and October 7 under MDY.""" + default = _observation("- [decision] @10/07/2026 The cutover ran.") + [assertion] = default.temporal + assert assertion.valid_during.lower == "2026-07-10" + + monkeypatch.setenv("BASIC_MEMORY_DATE_ORDER", "MDY") + monkeypatch.setattr(config_module, "_CONFIG_CACHE", None) + assert ConfigManager().config.date_order == "MDY" + + reordered = _observation("- [decision] @10/07/2026 The cutover ran.") + [assertion] = reordered.temporal + assert assertion.valid_during.lower == "2026-10-07" + + +def test_configured_date_order_never_reinterprets_an_iso_date(monkeypatch): + """An ISO date is unambiguous, so the preference must not touch it.""" + monkeypatch.setenv("BASIC_MEMORY_DATE_ORDER", "MDY") + monkeypatch.setattr(config_module, "_CONFIG_CACHE", None) + + observation = _observation("- [decision] @2026-07-10 The cutover ran.") + + [assertion] = observation.temporal + assert assertion.valid_during.lower == "2026-07-10" + + +# --- The unknown-kind diagnostic --- + + +def test_unknown_kind_in_a_range_literal_reports_diagnostic_and_keeps_text(): + """`@asserted` is well-formed but names no kind this system understands.""" + observation = _refusal("- [decision] @asserted[2026-06-10,) The cache layer will use Redis.") + + assert "unknown temporal kind 'asserted'" in (observation.temporal_error or "") + # The diagnostic names the kinds that would have worked. + assert "effective" in (observation.temporal_error or "") + # Never silently dropped: the text is still searchable content. + assert observation.content.startswith("@asserted[2026-06-10,)") + + +def test_unknown_kind_in_a_point_reports_diagnostic_and_keeps_text(): + """The payload reads as a date, so the author is plainly naming a kind.""" + observation = _refusal("- [decision] @asserted:2026-06-10 The cache layer will use Redis.") + + assert "unknown temporal kind 'asserted'" in (observation.temporal_error or "") + assert observation.content.startswith("@asserted:2026-06-10") + + +def test_an_unknown_kind_with_an_unreadable_payload_is_left_alone(): + """`@todo:fix the thing` is prose, not a broken qualifier. + + The diagnostic is reserved for a payload that actually reads as time; without that, + reporting would fire on ordinary `@word:` markers. + """ + observation = _observation("- [decision] @todo:fix the cache layer") + + assert observation.temporal == [] + assert observation.temporal_error is None + assert observation.content.startswith("@todo:fix") + + +# --- Everything else is content, silently --- + + +@pytest.mark.parametrize( + ("line", "kept"), + [ + # A known kind glued to something that is not a range literal. + ("- [decision] @effective[2026-06-10 Use Redis.", "@effective[2026-06-10"), + # A range mixing the two axes. + ("- [decision] @effective[2026-06-10,2026-07-27T00:00:00Z) Use Redis.", "@effective["), + # A range that ends before it begins. + ("- [decision] @effective[2026-08-01,2026-06-10) Use Redis.", "@effective["), + # A date that the calendar does not have. + ("- [decision] @effective[2026-02-30,) Use Redis.", "@effective[2026-02-30,)"), + ("- [decision] @2026-02-30 Use Redis.", "@2026-02-30"), + # A timestamp the calendar does not have. Read leniently it would file 10:00 on + # the 13th of January, and every reindex would project that same wrong instant. + ("- [decision] @occurred:2026-13-01T10:00:00 Use Redis.", "@occurred:2026-13-01"), + # The same impossible month in the spellings the canonical shapes do not cover: + # a bare year-month, and a quoted space-separated timestamp. Both used to be + # peeled off the line *and* filed as a date in some other month. + ("- [decision] @occurred:2026-13 Use Redis.", "@occurred:2026-13"), + ( + '- [decision] @occurred:"2026-13-01 10:00:00" Use Redis.', + '@occurred:"2026-13-01 10:00:00"', + ), + # A moment that leaves the calendar once it is shifted to UTC. + ("- [decision] @effective[9999-12-31T23:59:59-05:00,) Use Redis.", "@effective["), + ("- [decision] @effective:9999-12-31T23:59:59-05:00 Use Redis.", "@effective:"), + # Trailing junk: one broken token, not a qualifier plus content. + ("- [decision] @effective[2026-06-10,2026-07-27)x Use Redis.", "@effective["), + # The same rule for the point form. A calendar date carrying an instant marker + # with no instant behind it used to be peeled off the line and filed as a bare + # date, so the author reached for a moment and the index recorded an open-ended + # day -- and reproduced it on every reindex. + ("- [decision] @occurred:2026-01-01T Use Redis.", "@occurred:2026-01-01T"), + ("- [decision] @occurred:2026-01-01Z Use Redis.", "@occurred:2026-01-01Z"), + ( + "- [decision] @occurred:2026-01-01+14:00 Use Redis.", + "@occurred:2026-01-01+14:00", + ), + # A stray keystroke that moved the date itself: June 10 was peeled off the line + # and filed as October 6. + ("- [decision] @occurred:2026-06-10x Use Redis.", "@occurred:2026-06-10x"), + ], +) +def test_a_payload_that_does_not_read_as_time_stays_content(line: str, kept: str): + """No warning about how someone wrote a date -- the token is simply not a qualifier.""" + observation = _observation(line) + + assert observation.temporal == [] + assert observation.temporal_error is None + assert observation.content.startswith(kept) + + +# --- One qualifier never costs the note its index --- + + +def test_a_qualifier_at_the_end_of_the_calendar_does_not_fail_the_note(): + """Whatever a qualifier says, the rest of the note still parses. + + `@effective:9999-12` used to build `date(10000, 1, 1)`; the `ValueError` escaped + `parse_authored_point` and `parse_temporal_qualifier` -- neither of which guards that + call -- into the markdown parser, so *the whole document* failed over one qualifier: + every other observation and relation on the page went with it. December 9999 is + representable as `[9999-12-01,)`, so it files like any other period, and the + instant beside it, which is not representable at all, is simply left as content. + """ + content = "\n".join( + [ + "## Observations", + "- [decision] @effective:9999-12 The cache layer will use Redis.", + "- [decision] @effective:9999 The contract holds all year.", + "- [decision] @effective[9999-12-31T23:59:59-05:00,) An unstorable moment.", + "- [note] An ordinary observation that must still index.", + "", + "## Relations", + "- relates_to [[Cache Layer]]", + ] + ) + + parsed = parse(content) + + month, year, unstorable, ordinary = parsed.observations + [month_assertion] = month.temporal + [year_assertion] = year.temporal + assert str(month_assertion.valid_during) == "[9999-12-01,)" + assert str(year_assertion.valid_during) == "[9999-01-01,)" + assert month.content == "The cache layer will use Redis." + assert year.content == "The contract holds all year." + # Unreadable, so never peeled: the line keeps its exact text and reports nothing. + assert unstorable.temporal == [] + assert unstorable.temporal_error is None + assert unstorable.content == "@effective[9999-12-31T23:59:59-05:00,) An unstorable moment." + # The rest of the note is what the crash used to take with it. + assert ordinary.content == "An ordinary observation that must still index." + assert [relation.target for relation in parsed.relations] == ["Cache Layer"] + + +def test_qualifier_with_nothing_to_qualify_stays_content(): + """Peeling it would leave an empty observation, which the plugin drops outright.""" + observation = _observation("- [decision] @effective[2026-06-10,2026-07-27)") + + assert observation.temporal == [] + assert observation.temporal_error is None + assert observation.content == "@effective[2026-06-10,2026-07-27)" + + +@pytest.mark.parametrize( + "line", + [ + "- [note] Contact paul@basicmemory.com about the cutover", + "- [note] Ping @paul before the cutover", + "- [note] @basicmemory.com is great", + "- [note] @someone(2026) filed the ticket", + "- [note] Email me at ops@example.com (urgent)", + "- [note] @ops@example.com owns the runbook", + "- [note] @paul reviewed the cutover", + ], +) +def test_non_qualifier_at_tokens_are_ordinary_content(line: str): + """`@` is common prose, and none of it may become a valid-time assertion.""" + observation = _observation(line) + + assert observation.temporal == [] + assert observation.temporal_error is None + + +# --- Acceptance 9 and 10: the two axes never convert into one another --- + + +def test_date_only_bounds_never_acquire_time_or_zone(): + """Acceptance 9: a calendar date stays a calendar date, with no false precision.""" + observation = _observation("- [decision] @effective[2026-06-10,2026-07-27) Use Redis.") + + [assertion] = observation.temporal + assert assertion.valid_during.axis is TemporalRangeAxis.DATE + assert assertion.valid_during.lower == "2026-06-10" + assert assertion.valid_during.upper == "2026-07-27" + assert "T" not in (assertion.valid_during.lower or "") + assert "Z" not in (assertion.valid_during.upper or "") + + +def test_a_date_point_never_becomes_midnight_utc(): + """The point form must not promote a date onto the instant axis either. + + Midnight in *which* zone is a question the author never answered, and answering it + for them would make a date query and an instant query disagree about this note. + """ + observation = _observation("- [decision] @effective:2026-06-10 Use Redis.") + + [assertion] = observation.temporal + assert assertion.valid_during.axis is TemporalRangeAxis.DATE + assert assertion.valid_during.lower == "2026-06-10" + assert "T00:00" not in str(assertion.valid_during) + + +def test_naive_timestamp_bounds_are_read_as_utc(): + """A timestamp with no offset is UTC, not a refusal. + + Both spellings of the same moment must produce the same stored bound, or a search + would answer differently depending on how the author punctuated it. + """ + naive = _observation("- [decision] @occurred[2026-07-27T18:42:00,) Cutover ran.") + explicit = _observation("- [decision] @occurred[2026-07-27T18:42:00Z,) Cutover ran.") + + [from_naive] = naive.temporal + [from_explicit] = explicit.temporal + assert naive.temporal_error is None + assert from_naive.valid_during == from_explicit.valid_during + assert from_naive.valid_during.lower == "2026-07-27T18:42:00.000000Z" + + +def test_instant_bounds_normalize_to_utc(): + """An offset bound names an instant, and is stored as that instant in UTC.""" + observation = _observation( + "- [decision] @occurred[2026-07-27T18:42:00+02:00,2026-07-28T00:00:00Z) Cutover ran." + ) + + [assertion] = observation.temporal + assert assertion.valid_during.lower == "2026-07-27T16:42:00.000000Z" + # The author's own text is what round-trips, offset and all. + assert assertion.source_text.startswith("@occurred[2026-07-27T18:42:00+02:00") + + +# --- Direct scanner contract --- + + +def test_scanner_returns_content_unchanged_when_nothing_is_attempted(): + """The scanner is a peel, not a rewrite: untouched content is returned as-is.""" + result = parse_temporal_qualifier("Plain observation content") + + assert result.content == "Plain observation content" + assert result.assertions == () + assert result.error is None + + +def test_scanner_takes_an_explicit_date_order(): + """A caller that already holds the config passes it instead of re-reading it.""" + result = parse_temporal_qualifier("@10/07/2026 The cutover ran.", date_order="MDY") + + [assertion] = result.assertions + assert assertion.valid_during.lower == "2026-10-07" + assert result.content == "The cutover ran." diff --git a/tests/mcp/clients/test_search_client_temporal.py b/tests/mcp/clients/test_search_client_temporal.py new file mode 100644 index 000000000..200c61eeb --- /dev/null +++ b/tests/mcp/clients/test_search_client_temporal.py @@ -0,0 +1,97 @@ +"""The valid-time version-skew guard in SearchClient (SPEC-82). + +`SearchQuery` ignores unknown fields, which is normally a harmless forward-compatibility +choice. For a valid-time filter it is not: a server predating SPEC-82 accepts the request +and returns *unfiltered* results, which include exactly the undated sources the filter +was asked to exclude. The caller cannot tell the difference by looking at them. + +The server therefore confirms explicitly that it ran the filter, and the client refuses a +response that does not carry that confirmation. +""" + +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from basic_memory.mcp.clients import SearchClient + +# A response body from a server that knows nothing about valid time. +LEGACY_PAYLOAD: dict[str, Any] = { + "results": [], + "current_page": 1, + "page_size": 10, + "total": 0, + "total_is_exact": True, + "has_more": False, +} + + +def _stub_call_query(monkeypatch, payload: dict[str, Any]) -> None: + mock_response = MagicMock() + mock_response.json.return_value = payload + + async def mock_call_query(client, url, **kwargs): + return mock_response + + monkeypatch.setattr("basic_memory.mcp.tools.utils.call_query", mock_call_query) + + +@pytest.mark.parametrize("field", ["valid_at", "valid_overlaps", "time_kind"]) +@pytest.mark.asyncio +async def test_unconfirmed_valid_time_filter_is_refused(monkeypatch, field: str): + """Every valid-time field triggers the check; none of them may pass unconfirmed.""" + _stub_call_query(monkeypatch, dict(LEGACY_PAYLOAD)) + client = SearchClient(MagicMock(), "proj-123") + + with pytest.raises(ValueError, match="did not apply the requested valid-time filter"): + await client.search({"text": "cache", field: "effective"}, page=1, page_size=10) + + +@pytest.mark.asyncio +async def test_explicitly_false_confirmation_is_also_refused(monkeypatch): + """A server that answers "no" is as unusable as one that answers nothing.""" + _stub_call_query(monkeypatch, dict(LEGACY_PAYLOAD, temporal_applied=False)) + client = SearchClient(MagicMock(), "proj-123") + + with pytest.raises(ValueError, match="did not apply the requested valid-time filter"): + await client.search({"text": "cache", "valid_at": "2026-07-28"}, page=1, page_size=10) + + +@pytest.mark.asyncio +async def test_confirmed_valid_time_filter_is_accepted(monkeypatch): + _stub_call_query(monkeypatch, dict(LEGACY_PAYLOAD, temporal_applied=True)) + client = SearchClient(MagicMock(), "proj-123") + + response = await client.search( + {"text": "cache", "valid_at": "2026-07-28"}, page=1, page_size=10 + ) + + assert response.temporal_applied is True + + +@pytest.mark.asyncio +async def test_search_without_a_valid_time_filter_is_unaffected(monkeypatch): + """The guard must not touch ordinary searches against any server version.""" + _stub_call_query(monkeypatch, dict(LEGACY_PAYLOAD)) + client = SearchClient(MagicMock(), "proj-123") + + response = await client.search({"text": "cache"}, page=1, page_size=10) + + assert response.temporal_applied is None + assert response.total == 0 + + +@pytest.mark.asyncio +async def test_empty_valid_time_values_do_not_trigger_the_guard(monkeypatch): + """Fields present but unset are not a request, so nothing needs confirming.""" + _stub_call_query(monkeypatch, dict(LEGACY_PAYLOAD)) + client = SearchClient(MagicMock(), "proj-123") + + response = await client.search( + {"text": "cache", "valid_at": None, "valid_overlaps": None, "time_kind": None}, + page=1, + page_size=10, + ) + + assert response.temporal_applied is None diff --git a/tests/mcp/test_tool_contracts.py b/tests/mcp/test_tool_contracts.py index 758d75d1b..6af562a37 100644 --- a/tests/mcp/test_tool_contracts.py +++ b/tests/mcp/test_tool_contracts.py @@ -136,6 +136,9 @@ "tags", "status", "min_similarity", + "valid_at", + "valid_overlaps", + "time_kind", ], "tail": ["timeframe", "lines", "project", "project_id"], "view_note": ["identifier", "project", "project_id"], diff --git a/tests/mcp/test_tool_search_temporal.py b/tests/mcp/test_tool_search_temporal.py new file mode 100644 index 000000000..4aa3e2c72 --- /dev/null +++ b/tests/mcp/test_tool_search_temporal.py @@ -0,0 +1,368 @@ +"""End-to-end valid-time search through the MCP `search_notes` tool (SPEC-82). + +These tests exercise the whole chain the spec's acceptance cases describe: markdown +carrying temporal qualifiers is written through `write_note`, indexed, projected, and +then queried by authored valid time through `search_notes`. + +The scenario is the spec's own: one note holding two `[decision]` observations that +disagree about the cache layer, each qualified with the window it was effective over. +Both live in a *single* note on purpose -- that is what makes entity-granular filtering +insufficient and forces the projection to address individual observations. +""" + +import inspect +from typing import Any + +import pytest + +from basic_memory.mcp.tools import write_note +from basic_memory.mcp.tools.search import search_notes + +# The spec's worked example, verbatim: one note, two decisions, adjacent half-open +# effective windows meeting at the July 27 cutover. +CACHE_LAYER_NOTE = """\ +# Cache Layer + +## Observations +- [decision] @effective[2026-06-10,2026-07-27) The cache layer will use Redis. +- [decision] @effective[2026-07-27,) The cache layer will use Memcached. +""" + +# The same two decisions, written the convenient way. `@effective:2026-07-27` denotes +# `[2026-07-27,)` -- from the cutover onward -- so the cutover answers must not change. +CACHE_LAYER_POINT_NOTE = """\ +# Cache Layer + +## Observations +- [decision] @effective[2026-06-10,2026-07-27) The cache layer will use Redis. +- [decision] @effective:2026-07-27 The cache layer will use Memcached. +""" + +UNDATED_NOTE = """\ +# Queue Layer + +## Observations +- [decision] The queue layer will use RabbitMQ. +""" + + +async def _write_cache_layer_note(project_name: str) -> None: + await write_note( + project=project_name, + title="Cache Layer", + directory="decisions", + content=CACHE_LAYER_NOTE, + ) + + +def _contents(response: dict[str, Any]) -> list[str]: + """The matched observation text of every result, for readable assertions.""" + return [result["content"] or "" for result in response["results"]] + + +@pytest.mark.asyncio +async def test_valid_at_after_cutover_returns_memcached_excludes_redis(client, test_project): + """Acceptance 5: `valid_at=2026-07-28` returns Memcached and not Redis. + + July 28 falls inside `[2026-07-27,)` and outside `[2026-06-10,2026-07-27)`, whose + exclusive upper bound expires it exactly at the cutover. + """ + await _write_cache_layer_note(test_project.name) + + response = await search_notes( + project=test_project.name, + query="cache layer", + time_kind="effective", + valid_at="2026-07-28", + output_format="json", + ) + + assert isinstance(response, dict), response + contents = _contents(response) + assert any("Memcached" in content for content in contents), contents + assert not any("Redis" in content for content in contents), contents + + +@pytest.mark.asyncio +async def test_valid_at_before_cutover_returns_redis_excludes_memcached(client, test_project): + """Acceptance 6: `valid_at=2026-07-01` returns Redis and not Memcached.""" + await _write_cache_layer_note(test_project.name) + + response = await search_notes( + project=test_project.name, + query="cache layer", + time_kind="effective", + valid_at="2026-07-01", + output_format="json", + ) + + assert isinstance(response, dict), response + contents = _contents(response) + assert any("Redis" in content for content in contents), contents + assert not any("Memcached" in content for content in contents), contents + + +@pytest.mark.asyncio +async def test_point_qualifier_answers_the_cutover_like_a_range(client, test_project): + """The convenient form reaches the index and the predicate unchanged. + + `@effective:2026-07-27` means "from the cutover onward", so it must answer the + spec's two questions exactly as the explicit `[2026-07-27,)` range does -- and it + must not expire at midnight, which is what a closed single-day range would do. + """ + await write_note( + project=test_project.name, + title="Cache Layer", + directory="decisions", + content=CACHE_LAYER_POINT_NOTE, + ) + + after = await search_notes( + project=test_project.name, + query="cache layer", + time_kind="effective", + valid_at="2026-07-28", + output_format="json", + ) + before = await search_notes( + project=test_project.name, + query="cache layer", + time_kind="effective", + valid_at="2026-07-01", + output_format="json", + ) + + assert isinstance(after, dict) and isinstance(before, dict) + after_contents = _contents(after) + assert any("Memcached" in content for content in after_contents), after_contents + assert not any("Redis" in content for content in after_contents), after_contents + + before_contents = _contents(before) + assert any("Redis" in content for content in before_contents), before_contents + assert not any("Memcached" in content for content in before_contents), before_contents + + +@pytest.mark.asyncio +async def test_no_temporal_filter_lets_both_decisions_compete(client, test_project): + """Acceptance 7: with no valid-time filter both decisions are candidates again.""" + await _write_cache_layer_note(test_project.name) + + response = await search_notes( + project=test_project.name, + query="cache layer", + entity_types=["observation"], + output_format="json", + ) + + assert isinstance(response, dict), response + contents = _contents(response) + assert any("Redis" in content for content in contents), contents + assert any("Memcached" in content for content in contents), contents + # Ranking, not filtering, decides between them -- and nothing claims a filter ran. + assert response.get("temporal_applied") is None + + +@pytest.mark.asyncio +async def test_valid_overlaps_returns_both_decisions(client, test_project): + """A window spanning the cutover overlaps both effective ranges.""" + await _write_cache_layer_note(test_project.name) + + response = await search_notes( + project=test_project.name, + query="cache layer", + time_kind="effective", + valid_overlaps="[2026-06-01,2026-08-01)", + output_format="json", + ) + + assert isinstance(response, dict), response + contents = _contents(response) + assert any("Redis" in content for content in contents), contents + assert any("Memcached" in content for content in contents), contents + + +@pytest.mark.asyncio +async def test_undated_note_search_is_unchanged(client, test_project): + """Acceptance 1: a note with no qualifier searches exactly as it always did.""" + await write_note( + project=test_project.name, + title="Queue Layer", + directory="decisions", + content=UNDATED_NOTE, + ) + + response = await search_notes( + project=test_project.name, + query="RabbitMQ", + output_format="json", + ) + + assert isinstance(response, dict), response + assert response["results"], response + assert response.get("temporal_applied") is None + + +@pytest.mark.asyncio +async def test_valid_at_excludes_undated_observations(client, test_project): + """Acceptance 8: an undated statement cannot answer "what was true then".""" + await _write_cache_layer_note(test_project.name) + await write_note( + project=test_project.name, + title="Queue Layer", + directory="decisions", + content=UNDATED_NOTE, + ) + + unfiltered = await search_notes( + project=test_project.name, + query="layer", + entity_types=["observation"], + output_format="json", + ) + assert isinstance(unfiltered, dict), unfiltered + assert any("RabbitMQ" in content for content in _contents(unfiltered)) + + filtered = await search_notes( + project=test_project.name, + query="layer", + valid_at="2026-07-28", + output_format="json", + ) + assert isinstance(filtered, dict), filtered + assert not any("RabbitMQ" in content for content in _contents(filtered)) + assert filtered["temporal_applied"] is True + + +@pytest.mark.asyncio +async def test_results_carry_the_assertion_that_matched(client, test_project): + """A valid-time hit explains itself: kind, canonical range, and authored text.""" + await _write_cache_layer_note(test_project.name) + + response = await search_notes( + project=test_project.name, + query="cache layer", + valid_at="2026-07-28", + output_format="json", + ) + + assert isinstance(response, dict), response + [result] = [r for r in response["results"] if "Memcached" in (r["content"] or "")] + [assertion] = result["temporal"] + assert assertion["kind"] == "effective" + assert assertion["source_text"] == "@effective[2026-07-27,)" + assert assertion["valid_during"]["literal"] == "[2026-07-27,)" + assert assertion["valid_during"]["axis"] == "date" + assert assertion["valid_during"]["lower"] == "2026-07-27" + assert assertion["valid_during"]["lower_inclusive"] is True + # JSON output drops null fields, so an unbounded end shows up as an absent key. + assert assertion["valid_during"].get("upper") is None + + +@pytest.mark.asyncio +async def test_markdown_output_labels_the_time_kind(client, test_project): + """Human-readable output names the kind instead of printing a bare date.""" + await _write_cache_layer_note(test_project.name) + + rendered = await search_notes( + project=test_project.name, + query="cache layer", + valid_at="2026-07-28", + ) + + assert isinstance(rendered, str), rendered + assert "effective valid time: [2026-07-27,) (date)" in rendered + + +@pytest.mark.asyncio +async def test_kind_only_filter_finds_every_source_of_that_kind(client, test_project): + """A kind with no point or range is a legal question: who asserts this kind?""" + await _write_cache_layer_note(test_project.name) + await write_note( + project=test_project.name, + title="Queue Layer", + directory="decisions", + content=UNDATED_NOTE, + ) + + response = await search_notes( + project=test_project.name, + query="layer", + time_kind="effective", + output_format="json", + ) + + assert isinstance(response, dict), response + contents = _contents(response) + assert any("Redis" in content for content in contents), contents + assert any("Memcached" in content for content in contents), contents + assert not any("RabbitMQ" in content for content in contents), contents + + +@pytest.mark.asyncio +async def test_valid_at_and_valid_overlaps_together_are_refused(client, test_project): + """The two forms ask different questions; supplying both is an authoring error.""" + with pytest.raises(ValueError, match="not both"): + await search_notes( + project=test_project.name, + query="cache layer", + valid_at="2026-07-28", + valid_overlaps="[2026-06-01,2026-08-01)", + ) + + +@pytest.mark.asyncio +async def test_a_malformed_valid_time_filter_is_refused_rather_than_searched(client, test_project): + """A typo in a valid-time filter is an error, not a search that finds nothing.""" + await _write_cache_layer_note(test_project.name) + + with pytest.raises(ValueError, match="2026-13-01"): + await search_notes( + project=test_project.name, + query="cache layer", + valid_at="2026-13-01", + output_format="json", + ) + + +@pytest.mark.asyncio +async def test_all_projects_search_refuses_a_malformed_filter_instead_of_reporting_nothing( + client, test_project +): + """The same typo across every project must not come back as "no matches found". + + Through the real API each per-project leg 400s on the bad bound and returns a + `# Search Failed` string, which the fan-out logs and skips as an unavailable project. + Skipping every project leaves an empty response that still claims the filter ran -- + an invalid query wearing the shape of a successful one. + """ + await _write_cache_layer_note(test_project.name) + + with pytest.raises(ValueError, match="2026-13-01"): + await search_notes( + query="cache layer", + search_all_projects=True, + valid_at="2026-13-01", + output_format="json", + ) + + +@pytest.mark.asyncio +async def test_time_kind_alone_is_enough_search_criteria(client, test_project): + """A valid-time filter is real criteria, so it must not trip the empty-query guard.""" + await _write_cache_layer_note(test_project.name) + + response = await search_notes( + project=test_project.name, + time_kind="effective", + output_format="json", + ) + + assert isinstance(response, dict), response + assert len(response["results"]) == 2 + + +def test_tool_help_documents_undated_exclusion(): + """Acceptance 8: the exclusion is documented where a caller will read it.""" + doc = inspect.getdoc(search_notes) or "" + assert "Sources with no temporal qualifier are excluded" in doc + assert "valid_at" in doc and "valid_overlaps" in doc and "time_kind" in doc diff --git a/tests/mcp/test_tool_telemetry.py b/tests/mcp/test_tool_telemetry.py index 2d8cf83d7..6338c5dce 100644 --- a/tests/mcp/test_tool_telemetry.py +++ b/tests/mcp/test_tool_telemetry.py @@ -174,6 +174,7 @@ async def test_search_notes_emits_root_operation_and_project_context( "has_filters": True, "has_tags_filter": True, "has_status_filter": False, + "has_temporal_filter": False, }, ) span_names = [name for name, _ in spans] diff --git a/tests/mcp/tools/test_search_notes_multi_project_temporal.py b/tests/mcp/tools/test_search_notes_multi_project_temporal.py new file mode 100644 index 000000000..85fe04963 --- /dev/null +++ b/tests/mcp/tools/test_search_notes_multi_project_temporal.py @@ -0,0 +1,205 @@ +"""All-projects search must carry the valid-time filter into every project (SPEC-82). + +`_search_all_projects` re-declares the whole filter surface in its own signature and then +calls `search_notes` once per project. A filter that is not repeated there is dropped for +every project at once, and the merged answer would quietly mix filtered and unfiltered +rows -- the worst shape this failure can take, because the result still looks like an +answer. +""" + +import importlib +from contextlib import asynccontextmanager +from typing import Any + +import pytest + +from basic_memory.schemas.search import SearchItemType, SearchResponse, SearchResult + +PROJECT_REFS = [ + {"project": "personal/main", "project_id": "11111111-1111-1111-1111-111111111111"}, + {"project": "team-paul/main", "project_id": "22222222-2222-2222-2222-222222222222"}, +] + + +@pytest.fixture +def cloud_routing(monkeypatch): + """Pin the routing signals so project ids are forwarded deterministically.""" + search_mod = importlib.import_module("basic_memory.mcp.tools.search") + monkeypatch.setattr(search_mod, "is_factory_mode", lambda: False) + monkeypatch.setattr(search_mod, "_explicit_routing", lambda: True) + monkeypatch.setattr(search_mod, "_force_local_mode", lambda: False) + monkeypatch.setattr(search_mod, "has_cloud_credentials", lambda config: True) + + +def _install_stub_client(monkeypatch, payloads: list[dict[str, Any]], refs) -> None: + """Route every per-project search into a stub that records its query payload.""" + clients_mod = importlib.import_module("basic_memory.mcp.clients") + search_mod = importlib.import_module("basic_memory.mcp.tools.search") + + class StubProject: + def __init__(self, name: str | None, external_id: str | None): + self.name = name or "main" + self.external_id = external_id or "local-main" + + @asynccontextmanager + async def fake_get_project_client(project=None, context=None, project_id=None): + yield object(), StubProject(project, project_id) + + async def fake_resolve_project_and_path(client, identifier, project=None, context=None): + return StubProject(project, None), identifier, False + + async def fake_load_search_project_refs(context=None): + return refs + + class MockSearchClient: + def __init__(self, client, project_id): + self.project_id = project_id + + async def search(self, payload, page, page_size): + payloads.append(payload) + return SearchResponse( + results=[ + SearchResult( + title="Cache Layer", + permalink="main/decisions/cache-layer", + content="The cache layer will use Memcached.", + type=SearchItemType.OBSERVATION, + score=0.5, + file_path="/main/decisions/cache-layer.md", + ) + ], + current_page=page, + page_size=page_size, + total=1, + temporal_applied=True, + ) + + monkeypatch.setattr(search_mod, "_load_search_project_refs", fake_load_search_project_refs) + monkeypatch.setattr(search_mod, "get_project_client", fake_get_project_client) + monkeypatch.setattr(search_mod, "resolve_project_and_path", fake_resolve_project_and_path) + monkeypatch.setattr(clients_mod, "SearchClient", MockSearchClient) + + +@pytest.mark.asyncio +async def test_all_projects_search_forwards_the_valid_time_filter(monkeypatch, cloud_routing): + """Every project is asked the same valid-time question, not just the first.""" + search_mod = importlib.import_module("basic_memory.mcp.tools.search") + payloads: list[dict[str, Any]] = [] + _install_stub_client(monkeypatch, payloads, PROJECT_REFS) + + result = await search_mod.search_notes( + query="cache layer", + search_all_projects=True, + time_kind="effective", + valid_at="2026-07-28", + output_format="json", + ) + + assert isinstance(result, dict) + assert len(payloads) == len(PROJECT_REFS) + for payload in payloads: + assert payload["valid_at"] == "2026-07-28" + assert payload["time_kind"] == "effective" + assert payload["valid_overlaps"] is None + # Every leg confirmed it ran the filter, so the merged answer confirms it too. + assert result["temporal_applied"] is True + + +@pytest.mark.asyncio +async def test_all_projects_search_forwards_an_overlap_filter(monkeypatch, cloud_routing): + payloads: list[dict[str, Any]] = [] + _install_stub_client(monkeypatch, payloads, PROJECT_REFS) + search_mod = importlib.import_module("basic_memory.mcp.tools.search") + + await search_mod.search_notes( + query="cache layer", + search_all_projects=True, + valid_overlaps="[2026-06-01,2026-08-01)", + output_format="json", + ) + + assert [payload["valid_overlaps"] for payload in payloads] == [ + "[2026-06-01,2026-08-01)", + "[2026-06-01,2026-08-01)", + ] + + +@pytest.mark.asyncio +async def test_all_projects_search_without_a_filter_claims_nothing(monkeypatch, cloud_routing): + """An ordinary all-projects search stays exactly the payload it always was.""" + payloads: list[dict[str, Any]] = [] + _install_stub_client(monkeypatch, payloads, PROJECT_REFS) + search_mod = importlib.import_module("basic_memory.mcp.tools.search") + + result = await search_mod.search_notes( + query="cache layer", + search_all_projects=True, + output_format="json", + ) + + assert isinstance(result, dict) + assert "temporal_applied" not in result + + +@pytest.mark.parametrize( + ("valid_at", "valid_overlaps", "time_kind", "bad_value"), + [ + ("2026-13-01", None, None, "2026-13-01"), + (None, "2026-06-10..2026-07-27", None, "2026-06-10..2026-07-27"), + (None, None, "asserted", "asserted"), + ], +) +@pytest.mark.asyncio +async def test_a_malformed_filter_is_refused_before_any_project_is_searched( + monkeypatch, + cloud_routing, + valid_at: str | None, + valid_overlaps: str | None, + time_kind: str | None, + bad_value: str, +): + """A typo must read as an error, never as an all-projects search with no matches. + + Each per-project leg turns the API's 400 into a `# Search Failed` string, which the + fan-out cannot tell from a project being unavailable: it logs it and skips on. With + every project skipped the merged answer is an empty success that still reports + `temporal_applied`, so a mistyped filter would come back as the plausible-looking + "no matches found" for a question that never ran anywhere. Client-side validation is + the only layer that can tell the two apart, so it runs once, before the fan-out. + """ + payloads: list[dict[str, Any]] = [] + _install_stub_client(monkeypatch, payloads, PROJECT_REFS) + search_mod = importlib.import_module("basic_memory.mcp.tools.search") + + with pytest.raises(ValueError, match=bad_value): + await search_mod.search_notes( + query="cache layer", + search_all_projects=True, + output_format="json", + valid_at=valid_at, + valid_overlaps=valid_overlaps, + time_kind=time_kind, + ) + + assert payloads == [] + + +@pytest.mark.asyncio +async def test_all_projects_search_with_no_projects_still_confirms_the_filter( + monkeypatch, cloud_routing +): + """Zero projects is an empty answer to the valid-time question, not an unfiltered one.""" + payloads: list[dict[str, Any]] = [] + _install_stub_client(monkeypatch, payloads, []) + search_mod = importlib.import_module("basic_memory.mcp.tools.search") + + result = await search_mod.search_notes( + query="cache layer", + search_all_projects=True, + valid_at="2026-07-28", + output_format="json", + ) + + assert isinstance(result, dict) + assert result["results"] == [] + assert result["temporal_applied"] is True diff --git a/tests/repository/test_hybrid_fusion.py b/tests/repository/test_hybrid_fusion.py index 4a438ae6b..05aa2e7d7 100644 --- a/tests/repository/test_hybrid_fusion.py +++ b/tests/repository/test_hybrid_fusion.py @@ -18,6 +18,7 @@ from basic_memory.repository.search_repository_base import FUSION_BONUS, SearchRepositoryBase from basic_memory.repository.search_trace import SearchTraceCollector from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode +from basic_memory.temporal import TemporalFilter @dataclass @@ -82,6 +83,7 @@ async def search( categories: Optional[list[str]] = None, metadata_filters: Optional[dict[str, Any]] = None, file_path_prefix: Optional[str] = None, + temporal: Optional[TemporalFilter] = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: Optional[float] = None, limit: int = 10, @@ -162,6 +164,7 @@ def _fake_embedding_provider() -> EmbeddingProvider: categories=None, metadata_filters=None, file_path_prefix=None, + temporal=None, limit=10, offset=0, ) diff --git a/tests/repository/test_memory_time_index_contract.py b/tests/repository/test_memory_time_index_contract.py new file mode 100644 index 000000000..7bcb9bff4 --- /dev/null +++ b/tests/repository/test_memory_time_index_contract.py @@ -0,0 +1,879 @@ +"""The valid-time search predicate, as one contract over both dialects (SPEC-82). + +Acceptance case 4 requires SQLite and PostgreSQL to answer identically. This module is +that contract, expressed once: every test here uses only dialect-neutral fixtures +(`search_repository`, `session_maker`, `test_project`), and the repo runs the whole +`tests/` tree twice -- plain for SQLite, and under `BASIC_MEMORY_TEST_POSTGRES=1` for +PostgreSQL via testcontainers. A divergence therefore fails this same suite on one of +the two runs rather than hiding in a backend-specific file. + +The stored ranges below cover the dimensions PostgreSQL's range operators distinguish: +each inclusivity combination, each unbounded side, the fully unbounded range, the empty +range, and a separate instant axis that must never mix with the date axis. Timestamps +are written as explicit constants, never as "now", so the answers are the same on every +run and on every machine. + +A second population covers the dimension a *discrete* domain adds: date ranges whose +authored bounds and whose sets of days come apart. Those cases are what the half-open +canonicalization in `basic_memory.temporal` exists for, and both dialects must agree +about them too. +""" + +from dataclasses import dataclass +from datetime import datetime, timezone + +import pytest +import pytest_asyncio +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from basic_memory import db +from basic_memory.models import Entity, MemoryTimeIndex, Observation +from basic_memory.models.project import Project +from basic_memory.repository.memory_time_index_repository import MemoryTimeIndexRepository +from basic_memory.repository.search_repository import SearchIndexRow +from basic_memory.schemas.search import SearchItemType +from basic_memory.temporal import ( + TemporalFilter, + TemporalPoint, + TemporalRange, + TemporalRangeAxis, + TimeKind, + parse_point, + parse_range_literal, +) + +DATE = TemporalRangeAxis.DATE +INSTANT = TemporalRangeAxis.INSTANT + +# Every observation shares this word so one FTS query returns the whole population and +# the temporal predicate is the only thing that narrows it. That also proves the +# predicate composes with a MATCH/bm25 query rather than only with a bare scan. +SHARED_TERM = "cachelayer" + + +@dataclass(frozen=True, slots=True) +class StoredAssertion: + """One authored assertion, and the label the expectations refer to it by.""" + + label: str + valid_during: TemporalRange + kind: TimeKind = TimeKind.EFFECTIVE + + +def _date_range(literal: str) -> TemporalRange: + return parse_range_literal(literal, axis=DATE) + + +def _instant_range(literal: str) -> TemporalRange: + return parse_range_literal(literal, axis=INSTANT) + + +# The population under test. Labels are the vocabulary of every expectation below. +STORED_ASSERTIONS: tuple[StoredAssertion, ...] = ( + StoredAssertion("closed_open", _date_range("[2026-06-10,2026-07-27)")), + StoredAssertion("open_closed", _date_range("(2026-06-10,2026-07-27]")), + StoredAssertion("closed_closed", _date_range("[2026-06-10,2026-07-27]")), + StoredAssertion("open_open", _date_range("(2026-06-10,2026-07-27)")), + StoredAssertion("from_cutover", _date_range("[2026-07-27,)")), + StoredAssertion("before_june", _date_range("(,2026-06-10)")), + StoredAssertion("always", TemporalRange(axis=DATE)), + StoredAssertion("empty", TemporalRange.empty(DATE)), + StoredAssertion( + "instant_window", + _instant_range("[2026-07-27T16:00:00Z,2026-07-27T18:00:00Z)"), + ), + StoredAssertion( + "instant_offset", + # Authored in +02:00; normalization must make it the UTC window [14:00,15:00). + _instant_range("[2026-07-27T16:00:00+02:00,2026-07-27T17:00:00+02:00)"), + ), + StoredAssertion("due_window", _date_range("[2026-06-10,2026-07-27)"), kind=TimeKind.DUE), + # --- The discrete population --- + # + # Filed as `occurred` time and dated in March so it never widens an expectation + # above, and so no bound collides with the January bookkeeping timestamps that + # `test_projection_rows_carry_only_authored_bounds` watches for. + # + # Only March 2, and only March 3: adjacent as authored bounds, disjoint as days. + # This is the pair the half-open canonical form exists to tell apart. + StoredAssertion("only_mar_02", _date_range("(2026-03-01,2026-03-03)"), kind=TimeKind.OCCURRED), + StoredAssertion("only_mar_03", _date_range("(2026-03-02,2026-03-04)"), kind=TimeKind.OCCURRED), + # Back-to-back half-open periods, the shape a sequence of effective windows takes. + StoredAssertion( + "half_open_first", _date_range("[2026-03-10,2026-03-12)"), kind=TimeKind.OCCURRED + ), + StoredAssertion( + "half_open_second", _date_range("[2026-03-12,2026-03-14)"), kind=TimeKind.OCCURRED + ), + # Closed periods written by an author who means "through the 22nd": they share it. + StoredAssertion("closed_first", _date_range("[2026-03-20,2026-03-22]"), kind=TimeKind.OCCURRED), + StoredAssertion( + "closed_second", _date_range("[2026-03-22,2026-03-24]"), kind=TimeKind.OCCURRED + ), + StoredAssertion("one_day", _date_range("[2026-03-30,2026-03-30]"), kind=TimeKind.OCCURRED), + # After the 5th and before the 6th there is no day, so this authored range is the + # empty range -- something only the discrete reading can see. + StoredAssertion("no_such_day", _date_range("(2026-03-05,2026-03-06)"), kind=TimeKind.OCCURRED), + # An instant range with a closed upper end, so the date rewrite is proven to stop + # at the date axis rather than pushing this endpoint forward by a day. + StoredAssertion( + "instant_closed", + _instant_range("[2026-07-27T20:00:00Z,2026-07-27T21:00:00Z]"), + kind=TimeKind.OCCURRED, + ), +) + +DATE_LABELS = frozenset( + stored.label + for stored in STORED_ASSERTIONS + if stored.valid_during.axis is DATE and stored.kind is TimeKind.EFFECTIVE +) +NON_EMPTY_DATE_LABELS = DATE_LABELS - {"empty"} + + +@pytest_asyncio.fixture +async def temporal_population( + search_repository, + session_maker: async_sessionmaker[AsyncSession], + test_project: Project, +) -> dict[int, str]: + """Index one observation per stored assertion and project its valid time. + + Returns the observation id -> label map the assertions read results through, so a + test never has to know which row id the database happened to mint. + """ + indexed_at = datetime(2026, 1, 1, tzinfo=timezone.utc) + labels_by_id: dict[int, str] = {} + + async with db.scoped_session(session_maker) as session: + entity = Entity( + project_id=test_project.id, + title="Cache Layer", + note_type="note", + permalink="decisions/cache-layer", + file_path="decisions/cache-layer.md", + content_type="text/markdown", + created_at=indexed_at, + updated_at=indexed_at, + ) + session.add(entity) + await session.flush() + entity_id = entity.id + + for stored in STORED_ASSERTIONS: + observation = Observation( + project_id=test_project.id, + entity_id=entity_id, + content=f"{SHARED_TERM} decision {stored.label}", + category="decision", + ) + session.add(observation) + await session.flush() + labels_by_id[observation.id] = stored.label + + session.add( + MemoryTimeIndex( + project_id=test_project.id, + entity_id=entity_id, + source_type=SearchItemType.OBSERVATION.value, + source_id=observation.id, + time_kind=stored.kind.value, + range_axis=stored.valid_during.axis.value, + lower_value=stored.valid_during.lower, + upper_value=stored.valid_during.upper, + lower_inclusive=stored.valid_during.lower_inclusive, + upper_inclusive=stored.valid_during.upper_inclusive, + is_empty=stored.valid_during.is_empty, + extractor="observation", + source_text=str(stored.valid_during), + ) + ) + + for observation_id, label in labels_by_id.items(): + await search_repository.index_item( + SearchIndexRow( + id=observation_id, + type=SearchItemType.OBSERVATION.value, + title=f"decision: {SHARED_TERM} {label}", + content_stems=f"{SHARED_TERM} decision {label}", + content_snippet=f"{SHARED_TERM} decision {label}", + permalink=f"decisions/cache-layer/observations/decision/{label}", + file_path="decisions/cache-layer.md", + category="decision", + entity_id=entity_id, + metadata={"tags": None}, + created_at=indexed_at, + updated_at=indexed_at, + project_id=test_project.id, + ) + ) + return labels_by_id + + +async def _matching_labels( + search_repository, + labels_by_id: dict[int, str], + temporal: TemporalFilter, + *, + search_text: str | None = SHARED_TERM, +) -> set[str]: + """Run one valid-time search and translate the hits back into labels.""" + results = await search_repository.search( + search_text=search_text, + search_item_types=[SearchItemType.OBSERVATION], + temporal=temporal, + limit=50, + ) + return {labels_by_id[result.id] for result in results} + + +# --- Acceptance 4: containment answers identically on both backends --- + + +@pytest.mark.parametrize( + ("at", "expected"), + [ + # Inclusive lower endpoints are owned; exclusive ones are not. + ("2026-06-10", {"closed_open", "closed_closed", "always"}), + # Interior points belong to every interval that spans them. + ("2026-07-01", {"closed_open", "open_closed", "closed_closed", "open_open", "always"}), + # The cutover: exclusive upper ends have already expired, inclusive ones have not, + # and the next period's inclusive lower end has begun. + ("2026-07-27", {"open_closed", "closed_closed", "from_cutover", "always"}), + # Before every bounded lower end: only the unbounded-below ranges remain. + ("2026-06-01", {"before_june", "always"}), + # After every bounded upper end: only the unbounded-above ranges remain. + ("2026-08-01", {"from_cutover", "always"}), + ], + ids=["inclusive-lower", "interior", "cutover", "before-all", "after-all"], +) +@pytest.mark.asyncio +async def test_containment_contract(search_repository, temporal_population, at, expected): + """`valid_at` returns exactly the ranges containing that date, on either backend.""" + matched = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(kind=TimeKind.EFFECTIVE, at=parse_point(at)), + ) + + assert matched == expected + # The empty range contains no point, ever. + assert "empty" not in matched + + +# --- Acceptance 4: overlap answers identically on both backends --- + + +@pytest.mark.parametrize( + ("literal", "expected"), + [ + # Adjacent half-open periods do not overlap: this is why `[a,b)` is the right + # shape for a sequence of effective windows. + ("[2026-07-27,2026-08-01)", {"open_closed", "closed_closed", "from_cutover", "always"}), + # A window spanning the whole timeline meets every non-empty range. + ("[2026-06-01,2026-08-01)", NON_EMPTY_DATE_LABELS), + # An exclusive query lower end does not own the shared endpoint either. + ("(2026-07-27,2026-08-01)", {"from_cutover", "always"}), + # Unbounded below: only ranges that start before the exclusive upper end. + ("(,2026-06-10)", {"before_june", "always"}), + # Unbounded above: only ranges that have not already ended. + ("[2026-08-01,)", {"from_cutover", "always"}), + # A single closed point behaves exactly like containment of that point. + ("[2026-06-10,2026-06-10]", {"closed_open", "closed_closed", "always"}), + ], + ids=[ + "adjacent-half-open", + "spanning-window", + "exclusive-lower", + "unbounded-lower", + "unbounded-upper", + "degenerate-point", + ], +) +@pytest.mark.asyncio +async def test_overlap_contract(search_repository, temporal_population, literal, expected): + """`valid_overlaps` returns exactly the ranges sharing a point with the window.""" + matched = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(kind=TimeKind.EFFECTIVE, overlaps=_date_range(literal)), + ) + + assert matched == expected + + +@pytest.mark.asyncio +async def test_overlap_with_fully_unbounded_window_matches_every_non_empty_range( + search_repository, temporal_population +): + """A window with no endpoints separates nothing, so only `empty` is excluded.""" + matched = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(kind=TimeKind.EFFECTIVE, overlaps=TemporalRange(axis=DATE)), + ) + + assert matched == NON_EMPTY_DATE_LABELS + + +@pytest.mark.asyncio +async def test_overlap_with_empty_window_matches_nothing(search_repository, temporal_population): + """PostgreSQL: nothing overlaps the empty range, not even the empty range.""" + matched = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(kind=TimeKind.EFFECTIVE, overlaps=TemporalRange.empty(DATE)), + ) + + assert matched == set() + + +@pytest.mark.asyncio +async def test_stored_empty_range_matches_no_query(search_repository, temporal_population): + """An empty stored range contains no points, so no containment query finds it.""" + for at in ("2026-06-10", "2026-07-01", "2026-08-01"): + matched = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(kind=TimeKind.EFFECTIVE, at=parse_point(at)), + ) + assert "empty" not in matched, at + + +# --- The discrete domain: authored bounds vs. the days they denote --- +# +# Calendar dates are discrete, so an authored bound is not the boundary of the set of +# days it delimits. `basic_memory.temporal` closes that gap by storing every date range +# half-open; these tests are what proves the SQL predicate then answers about *days* +# rather than about endpoint strings -- identically on both backends. + + +async def _occurred_overlaps(search_repository, labels_by_id, literal: str) -> set[str]: + return await _matching_labels( + search_repository, + labels_by_id, + TemporalFilter(kind=TimeKind.OCCURRED, overlaps=_date_range(literal)), + ) + + +async def _occurred_at(search_repository, labels_by_id, at: str) -> set[str]: + return await _matching_labels( + search_repository, + labels_by_id, + TemporalFilter(kind=TimeKind.OCCURRED, at=parse_point(at)), + ) + + +@pytest.mark.asyncio +async def test_date_ranges_that_share_no_day_do_not_overlap(search_repository, temporal_population): + """The case the half-open canonical form exists to get right. + + `(2026-03-01,2026-03-03)` holds only March 2 and `(2026-03-02,2026-03-04)` holds + only March 3, so the two share nothing. Yet each range's raw endpoints lie inside + the other's raw bounds, so comparing the bounds *as authored* reports an overlap + that does not exist. Canonicalized to `[2026-03-02,2026-03-03)` and + `[2026-03-03,2026-03-04)`, the same scalar comparison is right. + """ + assert await _occurred_overlaps( + search_repository, temporal_population, "(2026-03-01,2026-03-03)" + ) == {"only_mar_02"} + + assert await _occurred_overlaps( + search_repository, temporal_population, "(2026-03-02,2026-03-04)" + ) == {"only_mar_03"} + + # And each holds exactly the one day it names. + assert await _occurred_at(search_repository, temporal_population, "2026-03-02") == { + "only_mar_02" + } + assert await _occurred_at(search_repository, temporal_population, "2026-03-03") == { + "only_mar_03" + } + + +@pytest.mark.asyncio +async def test_adjacent_half_open_ranges_share_no_day(search_repository, temporal_population): + """`[a,b)` and `[b,c)` meet at b without sharing it -- the point of the shape.""" + assert await _occurred_overlaps( + search_repository, temporal_population, "[2026-03-10,2026-03-12)" + ) == {"half_open_first"} + + assert await _occurred_overlaps( + search_repository, temporal_population, "[2026-03-12,2026-03-14)" + ) == {"half_open_second"} + + # March 12 belongs to the second period alone. + assert await _occurred_at(search_repository, temporal_population, "2026-03-11") == { + "half_open_first" + } + assert await _occurred_at(search_repository, temporal_population, "2026-03-12") == { + "half_open_second" + } + + +@pytest.mark.asyncio +async def test_closed_ranges_sharing_an_endpoint_do_overlap(search_repository, temporal_population): + """`[a,b]` and `[b,c]` both contain b, so they overlap on that one day. + + Canonicalization must preserve that: `[a,b+1)` and `[b,c+1)` still meet on b. An + author who writes closed bounds means the endpoint day is included, and the stored + form may not quietly take it away. + """ + assert await _occurred_overlaps( + search_repository, temporal_population, "[2026-03-20,2026-03-22]" + ) == {"closed_first", "closed_second"} + + # Narrowed to the shared day alone, both are still there. + assert await _occurred_at(search_repository, temporal_population, "2026-03-22") == { + "closed_first", + "closed_second", + } + assert await _occurred_at(search_repository, temporal_population, "2026-03-21") == { + "closed_first" + } + assert await _occurred_at(search_repository, temporal_population, "2026-03-23") == { + "closed_second" + } + + +@pytest.mark.asyncio +async def test_a_single_day_range_holds_exactly_that_day(search_repository, temporal_population): + """`[a,a]` is one day: neither empty, nor wider than the day the author wrote.""" + assert await _occurred_at(search_repository, temporal_population, "2026-03-30") == {"one_day"} + assert await _occurred_at(search_repository, temporal_population, "2026-03-29") == set() + assert await _occurred_at(search_repository, temporal_population, "2026-03-31") == set() + + assert await _occurred_overlaps( + search_repository, temporal_population, "[2026-03-30,2026-03-30]" + ) == {"one_day"} + + +@pytest.mark.asyncio +async def test_a_date_range_spanning_no_day_is_stored_empty( + search_repository, + session_maker: async_sessionmaker[AsyncSession], + temporal_population, + test_project: Project, +): + """`(2026-03-05,2026-03-06)` reads as an interval but names no day. + + Only the discrete reading can tell: as a continuous interval it looks like an + ordinary bounded range. Stored empty, it answers no question -- not even one about + the days on either side of it. + """ + for at in ("2026-03-05", "2026-03-06"): + assert "no_such_day" not in await _occurred_at( + search_repository, temporal_population, at + ), at + + assert await _occurred_overlaps( + search_repository, temporal_population, "[2026-03-01,2026-03-09)" + ) == {"only_mar_02", "only_mar_03"} + + repository = MemoryTimeIndexRepository(project_id=test_project.id) + async with db.scoped_session(session_maker) as session: + rows = await repository.find_for_sources( + session, + [(SearchItemType.OBSERVATION.value, source_id) for source_id in temporal_population], + ) + row = {temporal_population[row.source_id]: row for row in rows}["no_such_day"] + assert (row.is_empty, row.lower_value, row.upper_value) == (True, None, None) + + +@pytest.mark.asyncio +async def test_instant_ranges_are_untouched_by_the_date_canonicalization( + search_repository, temporal_population +): + """`instant_closed` is `[20:00Z,21:00Z]`, and stays exactly that. + + Instants are continuous: there is no next moment to close at, so the endpoint stays + owned and is emphatically not pushed forward by a day the way an inclusive date end + is. A query one microsecond past it, and one a whole day past it, both miss. + """ + at_upper = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(kind=TimeKind.OCCURRED, at=parse_point("2026-07-27T21:00:00Z")), + ) + assert at_upper == {"instant_closed"} + + for outside in ("2026-07-27T21:00:00.000001Z", "2026-07-28T21:00:00Z"): + missed = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(kind=TimeKind.OCCURRED, at=parse_point(outside)), + ) + assert missed == set(), outside + + +@pytest.mark.asyncio +async def test_projection_stores_date_bounds_in_the_canonical_half_open_form( + session_maker: async_sessionmaker[AsyncSession], + temporal_population, + test_project: Project, +): + """What actually lands in the columns the SQL predicate reads. + + The predicate compares bound values and inclusivity flags directly, so the + canonical form has to be in the rows -- not merely in the domain value that built + them. + """ + repository = MemoryTimeIndexRepository(project_id=test_project.id) + async with db.scoped_session(session_maker) as session: + rows = await repository.find_for_sources( + session, + [(SearchItemType.OBSERVATION.value, source_id) for source_id in temporal_population], + ) + by_label = {temporal_population[row.source_id]: row for row in rows} + + # Authored `(2026-03-01,2026-03-03)`: the exclusive lower end moved to the next day. + assert (by_label["only_mar_02"].lower_value, by_label["only_mar_02"].upper_value) == ( + "2026-03-02", + "2026-03-03", + ) + # Authored `[2026-03-20,2026-03-22]`: the inclusive upper end moved to the next day. + assert (by_label["closed_first"].lower_value, by_label["closed_first"].upper_value) == ( + "2026-03-20", + "2026-03-23", + ) + # Authored `[2026-03-30,2026-03-30]`: one day, spelled half-open. + assert (by_label["one_day"].lower_value, by_label["one_day"].upper_value) == ( + "2026-03-30", + "2026-03-31", + ) + for label in ("only_mar_02", "only_mar_03", "half_open_first", "closed_first", "one_day"): + row = by_label[label] + assert (row.lower_inclusive, row.upper_inclusive) == (True, False), label + + # The instant axis keeps the endpoint the author wrote, inclusivity and all. + instant = by_label["instant_closed"] + assert (instant.upper_value, instant.upper_inclusive) == ("2026-07-27T21:00:00.000000Z", True) + + +# --- Acceptance 9 and 10: the two axes are never confused --- + + +@pytest.mark.asyncio +async def test_date_query_does_not_match_instant_range(search_repository, temporal_population): + """Acceptance 9: a calendar-date question never reaches an instant range. + + Converting one into the other would have to invent a time of day or a timezone the + author never wrote, so the axes are simply disjoint. + """ + matched = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(kind=TimeKind.EFFECTIVE, at=parse_point("2026-07-27")), + ) + + assert "instant_window" not in matched + assert "instant_offset" not in matched + + +@pytest.mark.asyncio +async def test_instant_query_does_not_match_date_range(search_repository, temporal_population): + """The mirror image: an instant question never reaches a calendar-date range.""" + matched = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter( + kind=TimeKind.EFFECTIVE, + at=parse_point("2026-07-27T17:00:00Z"), + ), + ) + + assert matched == {"instant_window"} + assert not matched & DATE_LABELS + + +@pytest.mark.asyncio +async def test_instant_ranges_compare_as_instants_across_offsets( + search_repository, temporal_population +): + """Acceptance 10: an offset bound names an instant and is compared as one. + + `instant_offset` was authored as `[16:00+02:00,17:00+02:00)`, which is the UTC + window `[14:00Z,15:00Z)`. A UTC query point inside that window matches it; the same + clock reading interpreted naively would not. + """ + inside = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(kind=TimeKind.EFFECTIVE, at=parse_point("2026-07-27T14:30:00Z")), + ) + assert inside == {"instant_offset"} + + # 16:00 in +02:00 is 14:00Z, so the naive reading of the same digits is outside it. + outside = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(kind=TimeKind.EFFECTIVE, at=parse_point("2026-07-27T16:30:00Z")), + ) + assert outside == {"instant_window"} + + +@pytest.mark.asyncio +async def test_instant_endpoints_respect_inclusivity(search_repository, temporal_population): + """Instant bounds obey the same endpoint rules as dates, to the microsecond.""" + at_lower = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(kind=TimeKind.EFFECTIVE, at=parse_point("2026-07-27T16:00:00Z")), + ) + assert at_lower == {"instant_window"} + + at_upper = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(kind=TimeKind.EFFECTIVE, at=parse_point("2026-07-27T18:00:00Z")), + ) + assert at_upper == set() + + +# --- Kind narrowing --- + + +@pytest.mark.asyncio +async def test_kind_filter_narrows_to_one_kind(search_repository, temporal_population): + """Two kinds can assert the same interval; a kind filter separates them.""" + due = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(kind=TimeKind.DUE, at=parse_point("2026-07-01")), + ) + + assert due == {"due_window"} + + +@pytest.mark.asyncio +async def test_filter_without_a_kind_spans_every_kind(search_repository, temporal_population): + """Omitting the kind asks the question of every kind at once.""" + matched = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(at=parse_point("2026-07-01")), + ) + + assert matched == { + "closed_open", + "open_closed", + "closed_closed", + "open_open", + "always", + "due_window", + } + + +@pytest.mark.asyncio +async def test_kind_only_filter_selects_every_source_of_that_kind( + search_repository, temporal_population +): + """A kind with no window is a legal question, and the empty range still answers it. + + Without a window there is no axis to compare on and no interval to intersect, so + the filter asks only "does this source assert anything on this kind" -- which the + empty range does. + """ + matched = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(kind=TimeKind.EFFECTIVE), + ) + + assert matched == DATE_LABELS | {"instant_window", "instant_offset"} + + +# --- Acceptance 1 and 11: only authored bounds ever participate --- + + +@pytest.mark.asyncio +async def test_note_without_qualifier_writes_no_temporal_rows( + search_repository, + session_maker: async_sessionmaker[AsyncSession], + test_project: Project, + temporal_population, +): + """Acceptance 1: an undated observation is indexed, and projects no valid time.""" + indexed_at = datetime(2026, 1, 1, tzinfo=timezone.utc) + async with db.scoped_session(session_maker) as session: + entity = Entity( + project_id=test_project.id, + title="Queue Layer", + note_type="note", + permalink="decisions/queue-layer", + file_path="decisions/queue-layer.md", + content_type="text/markdown", + created_at=indexed_at, + updated_at=indexed_at, + ) + session.add(entity) + await session.flush() + observation = Observation( + project_id=test_project.id, + entity_id=entity.id, + content=f"{SHARED_TERM} undated decision", + category="decision", + ) + session.add(observation) + await session.flush() + undated_id = observation.id + undated_entity_id = entity.id + + await search_repository.index_item( + SearchIndexRow( + id=undated_id, + type=SearchItemType.OBSERVATION.value, + title="decision: undated", + content_stems=f"{SHARED_TERM} undated decision", + content_snippet=f"{SHARED_TERM} undated decision", + permalink="decisions/queue-layer/observations/decision/undated", + file_path="decisions/queue-layer.md", + category="decision", + entity_id=undated_entity_id, + metadata={"tags": None}, + created_at=indexed_at, + updated_at=indexed_at, + project_id=test_project.id, + ) + ) + + # Unfiltered, the undated observation is an ordinary hit. + unfiltered = await search_repository.search( + search_text=SHARED_TERM, + search_item_types=[SearchItemType.OBSERVATION], + limit=50, + ) + assert undated_id in {result.id for result in unfiltered} + + # Under any valid-time filter it is absent: it makes no claim to answer with. + filtered = await search_repository.search( + search_text=SHARED_TERM, + search_item_types=[SearchItemType.OBSERVATION], + temporal=TemporalFilter(at=parse_point("2026-07-01")), + limit=50, + ) + assert undated_id not in {result.id for result in filtered} + + +@pytest.mark.asyncio +async def test_projection_rows_carry_only_authored_bounds( + session_maker: async_sessionmaker[AsyncSession], + temporal_population, + test_project: Project, +): + """Acceptance 11: nothing but the authored qualifier reaches the stored bounds. + + Entity `created_at`/`updated_at` are deliberately January 1 while every authored + window is in June/July. If edit bookkeeping ever leaked into the projection, one of + these bounds would carry a January value. + """ + repository = MemoryTimeIndexRepository(project_id=test_project.id) + async with db.scoped_session(session_maker) as session: + rows = await repository.find_for_sources( + session, + [(SearchItemType.OBSERVATION.value, source_id) for source_id in temporal_population], + ) + + by_label = {temporal_population[row.source_id]: row for row in rows} + assert by_label["closed_open"].lower_value == "2026-06-10" + assert by_label["closed_open"].upper_value == "2026-07-27" + assert by_label["from_cutover"].upper_value is None + assert by_label["always"].lower_value is None and by_label["always"].upper_value is None + assert by_label["empty"].is_empty is True + for row in rows: + for bound in (row.lower_value, row.upper_value): + assert bound is None or not bound.startswith("2026-01"), row.source_text + + +# --- Pagination parity: filter and count must agree --- + + +@pytest.mark.asyncio +async def test_temporal_filter_count_matches_search(search_repository, temporal_population): + """`count()` runs the same predicate as `search()`, or pagination lies. + + The router gathers the two concurrently and derives `has_more` from the count, so a + count that ignored the filter would report pages that do not exist. + """ + temporal = TemporalFilter(kind=TimeKind.EFFECTIVE, at=parse_point("2026-07-01")) + results = await search_repository.search( + search_text=SHARED_TERM, + search_item_types=[SearchItemType.OBSERVATION], + temporal=temporal, + limit=50, + ) + total = await search_repository.count( + search_text=SHARED_TERM, + search_item_types=[SearchItemType.OBSERVATION], + temporal=temporal, + ) + + assert total == len(results) == 5 + + +@pytest.mark.asyncio +async def test_temporal_filter_applies_without_search_text(search_repository, temporal_population): + """A valid-time filter is criteria on its own; no MATCH is required to use it.""" + matched = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(kind=TimeKind.EFFECTIVE, at=parse_point("2026-08-01")), + search_text=None, + ) + + assert matched == {"from_cutover", "always"} + + +@pytest.mark.asyncio +async def test_temporal_filter_is_scoped_to_its_project( + search_repository, + session_maker: async_sessionmaker[AsyncSession], + project_repository, + temporal_population, +): + """Assertions belong to a project; another project's rows can never match here.""" + async with db.scoped_session(session_maker) as session: + other_project = await project_repository.create( + session, + { + "name": "other-project", + "description": "Isolation check", + "path": "/other/project", + "is_active": True, + "is_default": None, + }, + ) + + other_repository = type(search_repository)( + search_repository.session_maker, project_id=other_project.id + ) + results = await other_repository.search( + search_text=SHARED_TERM, + search_item_types=[SearchItemType.OBSERVATION], + temporal=TemporalFilter(kind=TimeKind.EFFECTIVE, at=parse_point("2026-07-01")), + limit=50, + ) + + assert results == [] + + +@pytest.mark.asyncio +async def test_temporal_point_and_range_agree_on_containment( + search_repository, temporal_population +): + """A point question is the degenerate closed range, so the two cannot disagree.""" + point = TemporalFilter(kind=TimeKind.EFFECTIVE, at=TemporalPoint(axis=DATE, value="2026-07-27")) + window = TemporalFilter( + kind=TimeKind.EFFECTIVE, + overlaps=TemporalRange( + axis=DATE, + lower="2026-07-27", + upper="2026-07-27", + lower_inclusive=True, + upper_inclusive=True, + ), + ) + + assert await _matching_labels( + search_repository, temporal_population, point + ) == await _matching_labels(search_repository, temporal_population, window) diff --git a/tests/repository/test_postgres_search_repository.py b/tests/repository/test_postgres_search_repository.py index 94c2e4c0a..e25b785e8 100644 --- a/tests/repository/test_postgres_search_repository.py +++ b/tests/repository/test_postgres_search_repository.py @@ -142,6 +142,10 @@ async def test_postgres_search_repository_index_and_search(session_maker, test_p permalink="docs/coffee-brewing", file_path="docs/coffee-brewing.md", type="entity", + # An entity row addresses itself: every indexing path sets entity_id on all three + # row kinds, and note_type is resolved through it, so a row built by hand here + # must carry it too or it belongs to no note at all. + entity_id=1, metadata={"note_type": "note"}, created_at=now, updated_at=now, diff --git a/tests/repository/test_rerank_pipeline.py b/tests/repository/test_rerank_pipeline.py index fcef636fa..5a496ea8b 100644 --- a/tests/repository/test_rerank_pipeline.py +++ b/tests/repository/test_rerank_pipeline.py @@ -794,6 +794,7 @@ async def deep_page(offset: int) -> list[SearchIndexRow]: categories=None, metadata_filters=None, file_path_prefix=None, + temporal=None, limit=1, offset=offset, ) diff --git a/tests/repository/test_search_trace.py b/tests/repository/test_search_trace.py index 88cd3772a..aa6f95a89 100644 --- a/tests/repository/test_search_trace.py +++ b/tests/repository/test_search_trace.py @@ -1392,6 +1392,7 @@ def test_non_text_criteria_and_null_owner_rows_stay_inspectable(): after_date=None, metadata_filters=None, file_path_prefix=None, + temporal=None, retrieval_mode=SearchRetrievalMode.FTS, min_similarity=None, ) diff --git a/tests/repository/test_semantic_search_base.py b/tests/repository/test_semantic_search_base.py index d6bc9aef0..e8154a4e8 100644 --- a/tests/repository/test_semantic_search_base.py +++ b/tests/repository/test_semantic_search_base.py @@ -33,6 +33,7 @@ ) from basic_memory.repository.semantic_vector_sync import PendingEmbeddingJob from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode +from basic_memory.temporal import TemporalFilter # --- Helpers --- @@ -88,6 +89,7 @@ async def search( categories: list[str] | None = None, metadata_filters: dict[str, Any] | None = None, file_path_prefix: str | None = None, + temporal: TemporalFilter | None = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: float | None = None, limit: int = 10, diff --git a/tests/repository/test_semantic_vector_sync.py b/tests/repository/test_semantic_vector_sync.py index c14e4cbd1..85c9ec721 100644 --- a/tests/repository/test_semantic_vector_sync.py +++ b/tests/repository/test_semantic_vector_sync.py @@ -16,6 +16,7 @@ from basic_memory.repository.search_trace import SearchTraceCollector from basic_memory.repository.semantic_chunking import VectorChunkRecord from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode +from basic_memory.temporal import TemporalFilter class _TestRepository(SearchRepositoryBase): @@ -57,6 +58,7 @@ async def search( categories: list[str] | None = None, metadata_filters: dict[str, Any] | None = None, file_path_prefix: str | None = None, + temporal: TemporalFilter | None = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: float | None = None, limit: int = 10, diff --git a/tests/repository/test_vector_pagination.py b/tests/repository/test_vector_pagination.py index 9e16a154a..8609324d3 100644 --- a/tests/repository/test_vector_pagination.py +++ b/tests/repository/test_vector_pagination.py @@ -16,6 +16,7 @@ from basic_memory.repository.search_index_row import SearchIndexRow from basic_memory.repository.search_trace import SearchTraceCollector from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode +from basic_memory.temporal import TemporalFilter @dataclass @@ -67,6 +68,7 @@ async def search( categories: list[str] | None = None, metadata_filters: dict[str, Any] | None = None, file_path_prefix: str | None = None, + temporal: TemporalFilter | None = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: float | None = None, limit: int = 10, @@ -194,6 +196,7 @@ async def run_page(offset, limit): categories=None, metadata_filters=None, file_path_prefix=None, + temporal=None, limit=limit, offset=offset, ) diff --git a/tests/repository/test_vector_temporal_filter.py b/tests/repository/test_vector_temporal_filter.py new file mode 100644 index 000000000..0270d7815 --- /dev/null +++ b/tests/repository/test_vector_temporal_filter.py @@ -0,0 +1,139 @@ +"""Valid-time filters reach the semantic retrieval modes too (SPEC-82). + +Vector and hybrid search do not evaluate the temporal predicate themselves: they build a +candidate set from embeddings and then intersect it with an FTS-mode pass that carries +every structured filter. That means a filter is only honored in those modes if it is +both *counted* as a requested filter and *forwarded* to the intersecting search. + +Missing either half fails silently -- semantic search would answer a valid-time question +with unfiltered results, including the undated sources the filter excludes. These tests +pin both halves at the seam rather than trusting the call sites to stay in step. +""" + +from typing import Any +from unittest.mock import AsyncMock, patch + +import pytest + +from basic_memory.temporal import TemporalFilter, TimeKind, parse_point +from tests.repository.test_hybrid_fusion import ( + HYBRID_KWARGS, + ConcreteSearchRepo as HybridSearchRepo, + FakeRow as HybridFakeRow, +) +from tests.repository.test_vector_threshold import ( + COMMON_SEARCH_KWARGS, + ConcreteSearchRepo as VectorSearchRepo, + FakeRow, + _fake_embedding_provider, + _make_vector_rows, + fake_scoped_session, +) + +TEMPORAL = TemporalFilter(kind=TimeKind.EFFECTIVE, at=parse_point("2026-07-28")) + + +def _vector_kwargs(**overrides: Any) -> dict[str, Any]: + return {**COMMON_SEARCH_KWARGS, **overrides} + + +def _hybrid_kwargs(**overrides: Any) -> dict[str, Any]: + return {**HYBRID_KWARGS, **overrides} + + +def _forwarded_temporal(leg: AsyncMock) -> Any: + """The `temporal` argument one retrieval leg was actually called with.""" + assert leg.await_args is not None, "leg was never awaited" + return leg.await_args.kwargs["temporal"] + + +@pytest.mark.asyncio +async def test_temporal_filter_applies_in_vector_mode(): + """A valid-time filter narrows the vector candidate set, and is forwarded verbatim.""" + repo = VectorSearchRepo() + repo._semantic_min_similarity = 0.0 + repo._embedding_provider = _fake_embedding_provider(AsyncMock(return_value=[0.0] * 384)) + + # The embedding neighbourhood offers three entities; only entity 1 asserts a range + # covering the queried date, so the FTS intersection pass returns just that one. + filter_pass = AsyncMock(return_value=[FakeRow(id=1)]) + + with ( + patch( + "basic_memory.repository.search_repository_base.db.scoped_session", fake_scoped_session + ), + patch.object(repo, "_ensure_vector_tables", new_callable=AsyncMock), + patch.object(repo, "_prepare_vector_session", new_callable=AsyncMock), + patch.object( + repo, + "_run_vector_query", + new_callable=AsyncMock, + return_value=_make_vector_rows([0.9, 0.8, 0.7]), + ), + patch.object( + repo, + "_fetch_search_index_rows_by_ids", + new_callable=AsyncMock, + return_value={("entity", i): FakeRow(id=i) for i in range(3)}, + ), + patch.object(repo, "search", filter_pass), + ): + results = await repo._search_vector_only(**_vector_kwargs(temporal=TEMPORAL)) + + assert [row.id for row in results] == [1] + # Counted as a requested filter... + filter_pass.assert_awaited_once() + # ...and forwarded unchanged, so the intersection asks the same question. + assert _forwarded_temporal(filter_pass) is TEMPORAL + + +@pytest.mark.asyncio +async def test_vector_mode_without_a_temporal_filter_runs_no_intersection_pass(): + """An unfiltered semantic search must not pay for a filter pass it does not need.""" + repo = VectorSearchRepo() + repo._semantic_min_similarity = 0.0 + repo._embedding_provider = _fake_embedding_provider(AsyncMock(return_value=[0.0] * 384)) + filter_pass = AsyncMock(return_value=[]) + + with ( + patch( + "basic_memory.repository.search_repository_base.db.scoped_session", fake_scoped_session + ), + patch.object(repo, "_ensure_vector_tables", new_callable=AsyncMock), + patch.object(repo, "_prepare_vector_session", new_callable=AsyncMock), + patch.object( + repo, + "_run_vector_query", + new_callable=AsyncMock, + return_value=_make_vector_rows([0.9]), + ), + patch.object( + repo, + "_fetch_search_index_rows_by_ids", + new_callable=AsyncMock, + return_value={("entity", 0): FakeRow(id=0)}, + ), + patch.object(repo, "search", filter_pass), + ): + results = await repo._search_vector_only(**_vector_kwargs()) + + assert [row.id for row in results] == [0] + filter_pass.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_temporal_filter_applies_in_hybrid_mode(): + """Hybrid fuses two legs; both must ask the same valid-time question.""" + repo = HybridSearchRepo() + fts_leg = AsyncMock(return_value=[HybridFakeRow(id=1, score=5.0, title="dated")]) + vector_leg = AsyncMock(return_value=[HybridFakeRow(id=1, score=0.9, title="dated")]) + + with ( + patch.object(repo, "search", fts_leg), + patch.object(repo, "_search_vector_only", vector_leg), + ): + results = await repo._search_hybrid(**_hybrid_kwargs(temporal=TEMPORAL)) + + assert [row.id for row in results] == [1] + assert _forwarded_temporal(fts_leg) is TEMPORAL + assert _forwarded_temporal(vector_leg) is TEMPORAL diff --git a/tests/repository/test_vector_threshold.py b/tests/repository/test_vector_threshold.py index 96b39549b..711f09c22 100644 --- a/tests/repository/test_vector_threshold.py +++ b/tests/repository/test_vector_threshold.py @@ -17,6 +17,7 @@ ) from basic_memory.repository.search_trace import SearchTraceCollector from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode +from basic_memory.temporal import TemporalFilter @dataclass @@ -71,6 +72,7 @@ async def search( categories: Optional[list[str]] = None, metadata_filters: Optional[dict[str, Any]] = None, file_path_prefix: Optional[str] = None, + temporal: Optional[TemporalFilter] = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: Optional[float] = None, limit: int = 10, @@ -166,6 +168,7 @@ async def fake_scoped_session(session_maker): categories=None, metadata_filters=None, file_path_prefix=None, + temporal=None, limit=10, offset=0, ) diff --git a/tests/schemas/test_document_agent_temporal.py b/tests/schemas/test_document_agent_temporal.py new file mode 100644 index 000000000..5460b5382 --- /dev/null +++ b/tests/schemas/test_document_agent_temporal.py @@ -0,0 +1,64 @@ +"""How the agent observation contract meets temporal qualifiers (SPEC-82). + +`DocumentAgentObservationV1` re-parses its own formatted markdown and requires the +parsed fields to match what it was given. Now that the parser peels a valid-time +qualifier off content, an untrusted agent that puts one inside `content` no longer +round-trips -- and is rejected. + +That is the intended MVP behavior, not an oversight: the agent contract has no temporal +field, so the alternative would be an agent silently minting valid-time assertions +through a text channel. Rejection is loud, and this test pins it so that adding +`temporal` to the agent contract later is a deliberate decision rather than an accident. +""" + +import pytest +from pydantic import ValidationError + +from basic_memory.schemas.document import DocumentAgentObservationV1 + + +def test_agent_observation_content_with_qualifier_is_rejected(): + """An agent cannot smuggle authored valid time through the content field.""" + with pytest.raises(ValidationError, match="must match parsed Markdown semantics"): + DocumentAgentObservationV1( + category="summary", + content="@effective[2026-06-10,2026-07-27) The cache layer will use Redis.", + ) + + +def test_agent_observation_with_a_malformed_qualifier_is_accepted_as_plain_text(): + """A refused qualifier is never peeled, so the line still round-trips exactly. + + This is the other half of "never silently dropped": text that only looks like a + qualifier stays content, and the agent contract keeps accepting it. + """ + observation = DocumentAgentObservationV1( + category="summary", + content="@asserted[2026-06-10,) The cache layer will use Redis.", + ) + + assert observation.content.startswith("@asserted[2026-06-10,)") + + +def test_ordinary_agent_observations_are_unaffected(): + """Acceptance 1 at the agent boundary: undated content behaves as it always did.""" + observation = DocumentAgentObservationV1( + category="summary", + content="The cache layer will use Redis.", + tags=("infra",), + context="agreed", + ) + + assert observation.content == "The cache layer will use Redis." + assert observation.tags == ("infra",) + assert observation.context == "agreed" + + +def test_email_addresses_in_agent_content_are_not_qualifiers(): + """`@` is common prose, and the contract must not start rejecting it.""" + observation = DocumentAgentObservationV1( + category="summary", + content="Contact paul@basicmemory.com about the cutover.", + ) + + assert "paul@basicmemory.com" in observation.content diff --git a/tests/services/test_search_service.py b/tests/services/test_search_service.py index 2fbb04610..99ed148b9 100644 --- a/tests/services/test_search_service.py +++ b/tests/services/test_search_service.py @@ -274,13 +274,32 @@ async def test_after_date_uses_updated_at(search_service): @pytest.mark.asyncio async def test_search_type(search_service, test_graph): - """Test search filters.""" - - # Should find only type + """`note_types` scopes results to notes of a type, not to entity rows. + + This test used to assert every result was an ENTITY row, which recorded a defect + rather than an intention: a note's type lives in its frontmatter, so only its entity + row carried it, and reading the type off each row dropped every observation and + relation belonging to the very same note. That is what made `note_types` unsatisfiable + together with a valid-time filter, since authored time lives on observation rows + (SPEC-82) -- the two predicates could not both be true of any row. + + Restricting *which kind* of row may match is `entity_types`' job, pinned by the test + directly below. The two axes are independent, and a query that sets neither returns + all three row kinds, so scoping by note type must not silently change the kinds. + """ results = await search_service.search(SearchQuery(note_types=["test"])) assert len(results) > 0 - for r in results: - assert r.type == SearchItemType.ENTITY + + # The three notes the fixture gives `note_type="test"`; "deep" and "deeper" differ. + typed_entity_ids = { + test_graph["root"].id, + test_graph["connected1"].id, + test_graph["connected2"].id, + } + # Every row belongs to a note of the requested type, whatever kind of row it is. + assert {r.entity_id for r in results} <= typed_entity_ids + # And rows other than the notes' own now survive the filter, which is the fix. + assert {r.type for r in results} - {SearchItemType.ENTITY} @pytest.mark.asyncio diff --git a/tests/services/test_search_service_temporal.py b/tests/services/test_search_service_temporal.py new file mode 100644 index 000000000..b1df63316 --- /dev/null +++ b/tests/services/test_search_service_temporal.py @@ -0,0 +1,314 @@ +"""Valid-time filtering through the search service (SPEC-82). + +The service is where the flat boundary strings become domain values, so this is where a +malformed filter must be refused loudly rather than degraded into a filter that quietly +matches something else. It is also the layer that proves acceptance case 11: a note's +edit bookkeeping is never reinterpreted as the time it claims to be true. +""" + +from datetime import datetime, timezone +from textwrap import dedent + +import pytest + +from basic_memory.schemas import Entity as EntitySchema +from basic_memory.schemas.search import SearchQuery +from basic_memory.services.search_service import ( + build_temporal_filter, + describe_search_criteria, +) +from basic_memory.temporal import TemporalQualifierError, TimeKind + +# The entity is created "now"; the qualifier claims June-July 2026. Keeping the two +# ranges disjoint is what makes acceptance case 11 testable at all. +EFFECTIVE_WINDOW_START = "2026-06-10" +EFFECTIVE_WINDOW_INSIDE = "2026-07-01" +EFFECTIVE_WINDOW_END = "2026-07-27" + +CACHE_LAYER_MARKDOWN = dedent(""" + # Cache Layer + + ## Observations + - [decision] @effective[2026-06-10,2026-07-27) The cache layer will use Redis. + """) + + +async def _index_cache_layer_note(entity_service, search_service): + """Create the dated note through the real write path and index it for search.""" + entity, _ = await entity_service.create_or_update_entity( + EntitySchema( + title="Cache Layer", + note_type="note", + directory="decisions", + content=CACHE_LAYER_MARKDOWN, + ) + ) + await search_service.index_entity(entity) + return entity + + +# --- Acceptance 11: entity time is never valid time --- + + +@pytest.mark.asyncio +async def test_entity_timestamps_are_never_used_as_observation_valid_time( + entity_service, search_service +): + """The note was written today; it claims to hold in June and July. + + Asking `valid_at` on the day the file was written must return nothing, because no + observation asserts that day. Asking inside the authored window returns the + observation. If edit bookkeeping ever leaked into the valid-time axis, the first + query would match and the distinction the spec draws would be gone. + """ + entity = await _index_cache_layer_note(entity_service, search_service) + written_on = entity.updated_at.date().isoformat() + assert written_on > EFFECTIVE_WINDOW_END, "fixture assumes the note is written after the window" + + at_write_time = await search_service.search( + SearchQuery(text="cache layer", valid_at=written_on) + ) + assert at_write_time == [] + + inside_window = await search_service.search( + SearchQuery(text="cache layer", valid_at=EFFECTIVE_WINDOW_INSIDE) + ) + assert [result.type for result in inside_window] == ["observation"] + assert "Redis" in (inside_window[0].content_snippet or "") + + +@pytest.mark.asyncio +async def test_after_date_still_filters_indexed_time_not_valid_time(entity_service, search_service): + """`after_date` keeps its meaning: it is the note's bookkeeping, not its claim. + + The note was indexed today and asserts a window that ended in July, so a filter on + each axis answers differently -- which is only possible because they stay separate. + """ + await _index_cache_layer_note(entity_service, search_service) + long_ago = datetime(2020, 1, 1, tzinfo=timezone.utc) + + recently_indexed = await search_service.search( + SearchQuery(text="cache layer", after_date=long_ago) + ) + assert recently_indexed + + still_effective_today = await search_service.search( + SearchQuery(text="cache layer", valid_at="2026-12-31") + ) + assert still_effective_today == [] + + +@pytest.mark.asyncio +async def test_valid_time_filter_narrows_to_the_asserting_observation( + entity_service, search_service +): + """A valid-time hit is the observation that carried the claim, not the whole note.""" + await _index_cache_layer_note(entity_service, search_service) + + results = await search_service.search( + SearchQuery(text="cache layer", time_kind="effective", valid_at=EFFECTIVE_WINDOW_START) + ) + + assert [result.type for result in results] == ["observation"] + + +@pytest.mark.asyncio +async def test_undated_note_is_excluded_by_a_valid_time_filter(entity_service, search_service): + """Acceptance 8, at the service layer: no claim means no answer.""" + entity, _ = await entity_service.create_or_update_entity( + EntitySchema( + title="Queue Layer", + note_type="note", + directory="decisions", + content="# Queue Layer\n\n## Observations\n- [decision] The queue layer uses RabbitMQ.\n", + ) + ) + await search_service.index_entity(entity) + + unfiltered = await search_service.search(SearchQuery(text="RabbitMQ")) + assert unfiltered + + filtered = await search_service.search( + SearchQuery(text="RabbitMQ", valid_at=EFFECTIVE_WINDOW_INSIDE) + ) + assert filtered == [] + + +# --- Diagnostics: the boundary refuses every malformed filter --- + + +def test_unknown_time_kind_is_refused_with_the_known_kinds(): + with pytest.raises(TemporalQualifierError, match="unknown time_kind 'asserted'") as exc_info: + build_temporal_filter(SearchQuery(text="cache", time_kind="asserted")) + + assert "effective" in str(exc_info.value) + + +def test_malformed_range_literal_is_refused(): + with pytest.raises(TemporalQualifierError, match="range literal must be"): + build_temporal_filter(SearchQuery(text="cache", valid_overlaps="2026-06-10..2026-07-27")) + + +def test_mixed_bound_kinds_are_refused(): + with pytest.raises(TemporalQualifierError, match="mix date-only and timestamp bounds"): + build_temporal_filter( + SearchQuery(text="cache", valid_overlaps="[2026-06-10,2026-07-27T00:00:00Z)") + ) + + +def test_timestamp_without_offset_is_read_as_utc(): + """A naive timestamp is not a rejection: it names the same instant as its `Z` form.""" + naive = build_temporal_filter(SearchQuery(text="cache", valid_at="2026-07-27T18:42:00")) + explicit = build_temporal_filter(SearchQuery(text="cache", valid_at="2026-07-27T18:42:00Z")) + + assert naive == explicit + assert naive is not None and naive.at is not None + assert naive.at.value == "2026-07-27T18:42:00.000000Z" + + +def test_impossible_range_is_refused(): + with pytest.raises(TemporalQualifierError, match="after upper bound"): + build_temporal_filter(SearchQuery(text="cache", valid_overlaps="[2026-08-01,2026-06-10)")) + + +def test_query_without_valid_time_fields_builds_no_filter(): + assert build_temporal_filter(SearchQuery(text="cache")) is None + + +def test_kind_only_query_builds_a_kind_filter(): + temporal = build_temporal_filter(SearchQuery(text="cache", time_kind="effective")) + + assert temporal is not None + assert temporal.kind is TimeKind.EFFECTIVE + assert temporal.at is None and temporal.overlaps is None + + +def test_valid_at_and_valid_overlaps_are_mutually_exclusive_at_the_schema(): + """The schema refuses the pair before any parsing or SQL can happen.""" + with pytest.raises(ValueError, match="not both"): + SearchQuery(text="cache", valid_at="2026-07-28", valid_overlaps="[2026-06-10,)") + + +# --- Query gating and traces --- + + +def test_a_valid_time_filter_alone_is_enough_criteria(): + """A temporal filter is real criteria; the empty-query guard must not swallow it.""" + assert SearchQuery(valid_at="2026-07-28").no_criteria() is False + assert SearchQuery(time_kind="effective").no_criteria() is False + assert SearchQuery(valid_overlaps="[2026-06-10,)").no_criteria() is False + assert SearchQuery().no_criteria() is True + + +@pytest.mark.asyncio +async def test_prepared_query_carries_the_parsed_filter(search_service): + prepared = search_service.prepare_query( + SearchQuery(text="cache", time_kind="effective", valid_at="2026-07-28") + ) + + assert prepared is not None + assert prepared.temporal is not None + assert prepared.temporal.kind is TimeKind.EFFECTIVE + assert prepared.temporal.at is not None + assert prepared.temporal.at.value == "2026-07-28" + + +@pytest.mark.asyncio +async def test_search_trace_describes_the_valid_time_question(search_service): + """A trace must show the question that actually ran, valid time included.""" + containment = search_service.prepare_query( + SearchQuery(text="cache", time_kind="effective", valid_at="2026-07-28") + ) + overlap = search_service.prepare_query( + SearchQuery(text="cache", valid_overlaps="[2026-06-10,2026-07-27)") + ) + plain = search_service.prepare_query(SearchQuery(text="cache")) + + assert containment is not None and overlap is not None and plain is not None + assert "temporal=kind=effective,valid_at=2026-07-28" in describe_search_criteria(containment) + assert "temporal=valid_overlaps=[2026-06-10,2026-07-27)" in describe_search_criteria(overlap) + assert "temporal=" not in describe_search_criteria(plain) + + +# --- Every authored assertion stays queryable by its own time --- + +TWICE_DATED_MARKDOWN = dedent(""" + # Cache Layer + + ## Observations + - [decision] @effective[2026-06-10,2026-07-27) The cache layer will use Redis. + - [decision] @effective[2027-06-10,2027-07-27) The cache layer will use Redis. + """) + +SECOND_WINDOW_INSIDE = "2027-07-01" + + +@pytest.mark.asyncio +async def test_same_statement_at_two_times_is_queryable_at_each(entity_service, search_service): + """One note, one sentence, two authored windows -- both must remain findable. + + The qualifier is peeled off before the observation is stored, so these two lines + persist identical content and derived identical synthetic permalinks. The search + index is keyed on permalink, so the second observation was skipped as a duplicate + while its temporal assertion went on addressing a row with no search projection: + querying 2027 returned nothing, and every reindex reproduced the omission from the + same markdown. The note says two things happened at two times; both must answer. + """ + entity, _ = await entity_service.create_or_update_entity( + EntitySchema( + title="Cache Layer Twice", + note_type="note", + directory="decisions", + content=TWICE_DATED_MARKDOWN, + ) + ) + await search_service.index_entity(entity) + + # The two rows are distinct statements and must carry distinct addresses. + first, second = entity.observations + assert first.permalink != second.permalink + + in_first = await search_service.search( + SearchQuery(text="cache layer", valid_at=EFFECTIVE_WINDOW_INSIDE) + ) + in_second = await search_service.search( + SearchQuery(text="cache layer", valid_at=SECOND_WINDOW_INSIDE) + ) + + assert [result.id for result in in_first] == [first.id] + assert [result.id for result in in_second] == [second.id] + # Each window answers with exactly one of them, never the same row twice. + assert first.id != second.id + + +@pytest.mark.asyncio +async def test_a_valid_time_query_can_also_scope_by_note_type(entity_service, search_service): + """Valid time selects observation rows; note type must not then exclude them. + + A note's type lives in its frontmatter, so only its entity row carries it. Reading the + type off each row made these two filters contradict each other -- every row the + temporal predicate admitted, the note-type predicate rejected -- so the conjunction + returned nothing however well the note matched. Resolving the type through the owning + note is what lets both questions be asked at once. + """ + entity = await _index_cache_layer_note(entity_service, search_service) + + scoped = await search_service.search( + SearchQuery( + text="cache layer", + valid_at=EFFECTIVE_WINDOW_INSIDE, + note_types=["note"], + ) + ) + + assert [result.type for result in scoped] == ["observation"] + assert scoped[0].entity_id == entity.id + # A type the note does not have still excludes it, so the filter is doing real work. + unscoped = await search_service.search( + SearchQuery( + text="cache layer", + valid_at=EFFECTIVE_WINDOW_INSIDE, + note_types=["conversation"], + ) + ) + assert unscoped == [] diff --git a/tests/test_config.py b/tests/test_config.py index c2f1588ca..c1e761287 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -5,8 +5,9 @@ import tempfile import pytest from datetime import datetime -from typing import Any, cast +from typing import Any, cast, get_args +from basic_memory.cli.commands.config import CONFIGURABLE_FIELDS from basic_memory.config import ( BasicMemoryConfig, ConfigManager, @@ -15,6 +16,7 @@ default_fastembed_cache_dir, resolve_data_dir, ) +from basic_memory.temporal import DEFAULT_DATE_ORDER, DateOrder from pathlib import Path @@ -1261,6 +1263,36 @@ def test_default_search_type_rejects_invalid_values(self): BasicMemoryConfig(default_search_type="invalid") +class TestDateOrderConfig: + """The preference used to read an ambiguous authored date (SPEC-82).""" + + def test_date_order_defaults_to_iso(self): + assert BasicMemoryConfig().date_order == "YMD" + + def test_date_order_accepts_the_three_component_orders(self): + for date_order in ("YMD", "DMY", "MDY"): + assert BasicMemoryConfig(date_order=date_order).date_order == date_order + + def test_date_order_rejects_anything_else(self): + with pytest.raises(Exception): + BasicMemoryConfig(date_order="ISO") + + def test_date_order_matches_the_domain_alias(self): + """The field is spelled as a bare Literal so `bm config set` can discover it. + + `temporal.DateOrder` is the same union used in function signatures; this pins + the two spellings together so neither can drift. + """ + assert set(get_args(BasicMemoryConfig.model_fields["date_order"].annotation)) == set( + get_args(DateOrder.__value__) + ) + assert BasicMemoryConfig().date_order == DEFAULT_DATE_ORDER + + def test_date_order_is_settable_from_the_cli(self): + """A user-facing preference is worth nothing if `bm config set` cannot reach it.""" + assert "date_order" in CONFIGURABLE_FIELDS + + class TestFormattingConfig: """Test file formatting configuration options.""" diff --git a/tests/test_memory_time_index_migration.py b/tests/test_memory_time_index_migration.py new file mode 100644 index 000000000..36dcc99bd --- /dev/null +++ b/tests/test_memory_time_index_migration.py @@ -0,0 +1,283 @@ +"""Migration coverage for the memory_time_index table (SPEC-82). + +The migration is deliberately dialect-neutral: every type it uses renders on SQLite and +PostgreSQL alike, so there is no branch to test per backend. What must be proven is +that the *same* definition arrives intact on both -- the columns, the cascade, the +lookup index, and the three CHECK constraints that keep an impossible range out of the +projection in the first place. + +Two halves, following the repo's established split: a real SQLite upgrade/downgrade +round trip, and an offline render of the same migration against the PostgreSQL dialect. +""" + +import io +import sqlite3 +from importlib import import_module +from typing import Any + +import pytest +from alembic import command +from alembic.migration import MigrationContext +from alembic.operations import Operations + +from tests.test_note_content_migration import sqlite_alembic_config + +migration = import_module("basic_memory.alembic.versions.u4t5e6m7p8o9_add_memory_time_index_table") + +# Pin the downgrade target to this migration's own parent. A relative "-1" would instead +# undo whichever migration currently sits at head, so the test would break every time a +# later revision lands. +DOWN_REVISION: str = str(migration.down_revision) + +EXPECTED_COLUMNS = { + "id", + "project_id", + "entity_id", + "source_type", + "source_id", + "time_kind", + "range_axis", + "lower_value", + "upper_value", + "lower_inclusive", + "upper_inclusive", + "is_empty", + "extractor", + "source_text", + "assertion_metadata", +} + +# One row per column, in the table's declared order, for the constraint probes below. +VALID_ROW = ( + 1, # project_id + 1, # entity_id + "observation", + 1, # source_id + "effective", + "date", + "2026-06-10", + "2026-07-27", + 1, # lower_inclusive + 0, # upper_inclusive + 0, # is_empty + "observation", + "@effective[2026-06-10,2026-07-27)", +) +INSERT_SQL = """ + INSERT INTO memory_time_index ( + project_id, entity_id, source_type, source_id, time_kind, range_axis, + lower_value, upper_value, lower_inclusive, upper_inclusive, is_empty, + extractor, source_text + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +""" + + +def _upgraded_database(tmp_path, monkeypatch, name: str): + """Run Alembic to head against a fresh temporary SQLite database.""" + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("BASIC_MEMORY_HOME", str(tmp_path / "basic-memory")) + database_path = tmp_path / name + config = sqlite_alembic_config(database_path) + command.upgrade(config, "head") + return database_path, config + + +def _row_with(**overrides: Any) -> tuple[Any, ...]: + """One valid row with named columns replaced, for the constraint probes.""" + columns = [ + "project_id", + "entity_id", + "source_type", + "source_id", + "time_kind", + "range_axis", + "lower_value", + "upper_value", + "lower_inclusive", + "upper_inclusive", + "is_empty", + "extractor", + "source_text", + ] + values = dict(zip(columns, VALID_ROW)) + values.update(overrides) + return tuple(values[column] for column in columns) + + +def _seed_parent_rows(connection: sqlite3.Connection) -> None: + """Insert the project and entity the projection rows below hang off.""" + connection.execute( + "INSERT INTO project (id, external_id, name, permalink, path, is_active," + " created_at, updated_at)" + " VALUES (1, 'project-1', 'p', 'p', '/p', 1, '2026-01-01', '2026-01-01')" + ) + connection.execute( + "INSERT INTO entity (id, external_id, project_id, title, note_type, permalink," + " file_path, content_type, created_at, updated_at)" + " VALUES (1, 'entity-1', 1, 't', 'note', 'p/t', 't.md', 'text/markdown'," + " '2026-01-01', '2026-01-01')" + ) + + +def test_alembic_upgrade_creates_memory_time_index_table(tmp_path, monkeypatch): + """Upgrading to head creates the projection table with its full contract.""" + database_path, _ = _upgraded_database(tmp_path, monkeypatch, "memory-time-index.db") + + connection = sqlite3.connect(database_path) + try: + columns = { + row[1] for row in connection.execute("PRAGMA table_info(memory_time_index)").fetchall() + } + assert columns == EXPECTED_COLUMNS + + foreign_keys = connection.execute("PRAGMA foreign_key_list(memory_time_index)").fetchall() + entity_fk = next(row for row in foreign_keys if row[3] == "entity_id") + project_fk = next(row for row in foreign_keys if row[3] == "project_id") + assert (entity_fk[2], entity_fk[4]) == ("entity", "id") + # Valid time is removed with the entity it was asserted about. + assert entity_fk[6].upper() == "CASCADE" + assert (project_fk[2], project_fk[4]) == ("project", "id") + # source_id addresses whichever table source_type names, so it carries no FK. + assert {row[3] for row in foreign_keys} == {"entity_id", "project_id"} + + indexes = { + row[1] for row in connection.execute("PRAGMA index_list(memory_time_index)").fetchall() + } + assert "ix_memory_time_index_lookup" in indexes + assert "ix_memory_time_index_entity_id" in indexes + + lookup_columns = [ + row[2] + for row in connection.execute( + "PRAGMA index_info(ix_memory_time_index_lookup)" + ).fetchall() + ] + # The predicate filters on project/kind/axis and projects (source_type, source_id), + # so this one index both drives the scan and covers its output. + assert lookup_columns == [ + "project_id", + "time_kind", + "range_axis", + "source_type", + "source_id", + ] + + # Bound values are deliberately unindexed: the full-text candidate set drives. + assert not any(index.startswith("ix_memory_time_index_lower") for index in indexes) + assert not any(index.startswith("ix_memory_time_index_upper") for index in indexes) + finally: + connection.close() + + +def test_upgraded_table_accepts_a_well_formed_assertion(tmp_path, monkeypatch): + """The CHECK constraints must not reject the rows the projection actually writes.""" + database_path, _ = _upgraded_database(tmp_path, monkeypatch, "memory-time-index-insert.db") + + connection = sqlite3.connect(database_path) + try: + _seed_parent_rows(connection) + connection.execute(INSERT_SQL, VALID_ROW) + # Unbounded and empty ranges are legal shapes, not edge cases. + connection.execute( + INSERT_SQL, + _row_with(source_id=2, lower_value=None, lower_inclusive=0), + ) + connection.execute( + INSERT_SQL, + _row_with( + source_id=3, + lower_value=None, + upper_value=None, + lower_inclusive=0, + upper_inclusive=0, + is_empty=1, + ), + ) + connection.commit() + + assert connection.execute("SELECT COUNT(*) FROM memory_time_index").fetchone()[0] == 3 + finally: + connection.close() + + +@pytest.mark.parametrize( + ("overrides", "constraint"), + [ + ({"range_axis": "week"}, "ck_memory_time_index_range_axis"), + # An empty range with endpoints would describe the same interval two ways. + ({"is_empty": 1}, "ck_memory_time_index_empty_has_no_bounds"), + # PostgreSQL's rule: there is no endpoint to include on an unbounded side. + ( + {"lower_value": None, "lower_inclusive": 1}, + "ck_memory_time_index_unbounded_is_exclusive", + ), + ], + ids=["unknown-axis", "empty-with-bounds", "unbounded-but-inclusive"], +) +def test_check_constraints_reject_impossible_rows(tmp_path, monkeypatch, overrides, constraint): + """An interval the domain cannot produce must not be storable either.""" + database_path, _ = _upgraded_database( + tmp_path, monkeypatch, f"memory-time-index-{constraint}.db" + ) + + connection = sqlite3.connect(database_path) + try: + _seed_parent_rows(connection) + with pytest.raises(sqlite3.IntegrityError, match=constraint): + connection.execute(INSERT_SQL, _row_with(**overrides)) + finally: + connection.close() + + +def test_alembic_downgrade_drops_memory_time_index_table(tmp_path, monkeypatch): + """Downgrading past this revision removes the table and both of its indexes.""" + database_path, config = _upgraded_database(tmp_path, monkeypatch, "memory-time-index-down.db") + command.downgrade(config, DOWN_REVISION) + + connection = sqlite3.connect(database_path) + try: + table_exists = connection.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'memory_time_index'" + ).fetchone() + remaining_indexes = connection.execute( + "SELECT name FROM sqlite_master WHERE type = 'index'" + " AND name LIKE 'ix_memory_time_index%'" + ).fetchall() + finally: + connection.close() + + assert table_exists is None + assert remaining_indexes == [] + + +def test_postgres_render_carries_the_same_definition(monkeypatch): + """The identical migration renders on PostgreSQL with no dialect branching. + + Rendering offline is what proves it: if any type, default, or constraint needed a + backend-specific spelling, this would fail here rather than on a deploy. + """ + buffer = io.StringIO() + context = MigrationContext.configure( + dialect_name="postgresql", + opts={"as_sql": True, "output_buffer": buffer}, + ) + monkeypatch.setattr(migration, "op", Operations(context)) + + migration.upgrade() + migration.downgrade() + + sql = buffer.getvalue() + assert "CREATE TABLE memory_time_index" in sql + assert "FOREIGN KEY(entity_id) REFERENCES entity (id) ON DELETE CASCADE" in sql + assert "ck_memory_time_index_range_axis" in sql + assert "ck_memory_time_index_empty_has_no_bounds" in sql + assert "ck_memory_time_index_unbounded_is_exclusive" in sql + assert ( + "CREATE INDEX ix_memory_time_index_lookup ON memory_time_index " + "(project_id, time_kind, range_axis, source_type, source_id)" in sql + ) + # Bounds stay portable text on both backends; a native range column would be a + # later, generated addition rather than a change to this definition. + assert "lower_value VARCHAR(32)" in sql + assert "upper_value VARCHAR(32)" in sql + assert "DROP TABLE memory_time_index" in sql diff --git a/tests/test_note_section_migration.py b/tests/test_note_section_migration.py index bc1ba0d3e..7370575ad 100644 --- a/tests/test_note_section_migration.py +++ b/tests/test_note_section_migration.py @@ -4,8 +4,16 @@ from alembic import command +from basic_memory.alembic.versions import ( # type: ignore[attr-defined] + t3n4o5t6e7s8_add_note_section_table as migration, +) from tests.test_note_content_migration import sqlite_alembic_config +# Pin the downgrade target to this migration's own parent. A relative "-1" would +# instead undo whichever migration currently sits at head, so the test would break +# every time a later revision lands. +DOWN_REVISION: str = str(migration.down_revision) + def test_alembic_upgrade_creates_note_section_table(tmp_path, monkeypatch): """Running Alembic head creates note_section with its expected contract.""" @@ -70,14 +78,14 @@ def test_alembic_upgrade_creates_note_section_table(tmp_path, monkeypatch): def test_alembic_downgrade_drops_note_section_table(tmp_path, monkeypatch): - """Downgrading one revision removes the table and its indexes.""" + """Downgrading past this revision removes the table and its indexes.""" monkeypatch.setenv("HOME", str(tmp_path)) monkeypatch.setenv("BASIC_MEMORY_HOME", str(tmp_path / "basic-memory")) database_path = tmp_path / "note-section-downgrade.db" config = sqlite_alembic_config(database_path) command.upgrade(config, "head") - command.downgrade(config, "-1") + command.downgrade(config, DOWN_REVISION) connection = sqlite3.connect(database_path) try: diff --git a/tests/test_temporal.py b/tests/test_temporal.py new file mode 100644 index 000000000..8605d2343 --- /dev/null +++ b/tests/test_temporal.py @@ -0,0 +1,1185 @@ +"""The portable temporal value types and their lexical grammar (SPEC-82). + +These values are the shared vocabulary between the markdown parser, the projection, and +both search dialects. Two properties carry the whole design and are pinned here: + +* Canonical bounds are fixed width, so byte-lexicographic order *is* chronological + order -- which is what lets one SQL predicate serve SQLite and PostgreSQL alike. +* Dates and instants are separate axes. A date never gains a time of day or a zone, and + an instant written without an offset is read as UTC -- the same convention the rest + of the codebase applies to naive datetimes. +""" + +from datetime import datetime, timedelta + +import pytest +from freezegun import freeze_time + +from basic_memory.temporal import ( + DEFAULT_DATE_ORDER, + TemporalAssertion, + TemporalFilter, + TemporalPoint, + TemporalQualifierError, + TemporalRange, + TemporalRangeAxis, + TimeKind, + canonical_bound, + parse_authored_point, + parse_point, + parse_range_literal, +) + +DATE = TemporalRangeAxis.DATE +INSTANT = TemporalRangeAxis.INSTANT + + +# --- Canonical bounds --- + + +@pytest.mark.parametrize( + ("written", "canonical"), + [ + ("2026-07-27T18:42:00Z", "2026-07-27T18:42:00.000000Z"), + ("2026-07-27t18:42:00z", "2026-07-27T18:42:00.000000Z"), + ("2026-07-27T18:42:00+02:00", "2026-07-27T16:42:00.000000Z"), + ("2026-07-27T18:42:00-05:00", "2026-07-27T23:42:00.000000Z"), + ("2026-07-27T18:42:00.5Z", "2026-07-27T18:42:00.500000Z"), + ("2026-07-27T18:42:00.123456Z", "2026-07-27T18:42:00.123456Z"), + ], +) +def test_instant_bounds_normalize_to_fixed_width_utc(written: str, canonical: str): + """Every instant lands on the same 27-character UTC form, whatever it was written as.""" + assert canonical_bound(written, INSTANT) == canonical + assert len(canonical) == 27 + + +def test_canonical_instants_sort_chronologically_as_plain_strings(): + """Fixed width plus fixed separator positions makes string order time order. + + This is the property the SQL predicate relies on: comparing canonical text columns + with `<` and `>` is comparing moments, on either backend, with no typed date bind. + """ + written = [ + "2026-07-27T18:42:00+02:00", # 16:42Z + "2026-07-27T17:00:00Z", + "2026-07-26T23:59:59Z", + "2026-07-27T18:42:00Z", + ] + canonical = [canonical_bound(bound, INSTANT) for bound in written] + + assert sorted(canonical) == [ + "2026-07-26T23:59:59.000000Z", + "2026-07-27T16:42:00.000000Z", + "2026-07-27T17:00:00.000000Z", + "2026-07-27T18:42:00.000000Z", + ] + + +def test_date_bounds_are_already_canonical(): + assert canonical_bound("2026-07-27", DATE) == "2026-07-27" + + +@pytest.mark.parametrize( + "bound", + [ + "20260727", # compact ISO: accepted by date.fromisoformat, breaks fixed width + "2026-7-27", + "27-07-2026", + "2026-02-30", + "not-a-date", + ], +) +def test_malformed_date_bounds_are_refused(bound: str): + with pytest.raises(TemporalQualifierError): + canonical_bound(bound, DATE) + + +@pytest.mark.parametrize( + ("written", "canonical"), + [ + ("2026-07-27T18:42:00", "2026-07-27T18:42:00.000000Z"), + ("2026-07-27t18:42:00", "2026-07-27T18:42:00.000000Z"), + ("2026-07-27T18:42:00.5", "2026-07-27T18:42:00.500000Z"), + ], +) +def test_naive_timestamp_bounds_are_read_as_utc(written: str, canonical: str): + """A timestamp with no offset is UTC, not an error. + + This is the house convention for every other naive datetime in the codebase, and + it is what lets an author write a timestamp without learning RFC 3339's offset + syntax first. + """ + assert canonical_bound(written, INSTANT) == canonical + + +@pytest.mark.parametrize( + "bound", + [ + "2026-07-27 18:42:00", # space separator: not the canonical bound shape + "2026-07-27T18:42", # no seconds + "2026-07-27T18:42:00.1234567Z", # finer than microseconds: would be truncated + "2026-07-27", + ], +) +def test_malformed_instant_bounds_are_refused(bound: str): + with pytest.raises(TemporalQualifierError): + canonical_bound(bound, INSTANT) + + +def test_sub_microsecond_precision_is_refused_rather_than_truncated(): + """Dropping digits would make the stored bound name a different instant.""" + with pytest.raises(TemporalQualifierError, match="microsecond precision"): + canonical_bound("2026-07-27T18:42:00.1234567Z", INSTANT) + + +def test_timestamp_shaped_bound_on_a_date_that_does_not_exist_is_refused(): + """The lexical shape admits `2026-02-30T...`; the calendar does not.""" + with pytest.raises(TemporalQualifierError, match="not a valid timestamp"): + canonical_bound("2026-02-30T10:00:00Z", INSTANT) + + +@pytest.mark.parametrize( + "bound", + [ + "9999-12-31T23:59:59-05:00", # 10000-01-01 in UTC + "0001-01-01T00:00:00+05:00", # year 0 in UTC + ], +) +def test_instant_bounds_that_leave_the_calendar_in_utc_are_refused(bound: str): + """Normalizing to UTC *moves* a moment, and the move can run off the calendar. + + Refused as a `TemporalQualifierError` like every other unreadable bound, which is + what makes it survivable: `datetime.astimezone` signals this with `OverflowError`, + and an `OverflowError` is not a `ValueError`, so it slipped past every handler above + -- failing a whole note's parse, or a whole search request, over one bound. + """ + with pytest.raises(TemporalQualifierError, match="leaves the calendar"): + canonical_bound(bound, INSTANT) + + +# --- TemporalPoint --- + + +def test_point_rejects_a_non_canonical_value(): + """A value that skipped canonicalization must not enter the domain.""" + with pytest.raises(TemporalQualifierError, match="not canonical"): + TemporalPoint(axis=INSTANT, value="2026-07-27T18:42:00Z") + + +def test_point_renders_its_canonical_value(): + assert str(TemporalPoint(axis=DATE, value="2026-07-27")) == "2026-07-27" + + +def test_parse_point_infers_the_axis_from_what_was_written(): + assert parse_point("2026-07-27") == TemporalPoint(axis=DATE, value="2026-07-27") + assert parse_point(" 2026-07-27T18:42:00+02:00 ") == TemporalPoint( + axis=INSTANT, value="2026-07-27T16:42:00.000000Z" + ) + + +def test_parse_point_refuses_an_empty_string(): + with pytest.raises(TemporalQualifierError, match="must not be empty"): + parse_point(" ") + + +def test_parse_point_reads_a_naive_timestamp_as_utc(): + """The search boundary follows the same naive-is-UTC rule as authored bounds.""" + assert parse_point("2026-07-27T18:42:00") == TemporalPoint( + axis=INSTANT, value="2026-07-27T18:42:00.000000Z" + ) + + +# --- Flexible authored points --- +# +# The convenient form. `parse_authored_point` reads whatever dateparser reads and +# canonicalizes it into a TemporalRange, so an author never has to spell out a range +# literal to say when something started. + + +@pytest.mark.parametrize( + ("written", "literal", "axis"), + [ + # A year and a month are periods the author delimited by writing them. + ("2026", "[2026-01-01,2027-01-01)", DATE), + ("2026-06", "[2026-06-01,2026-07-01)", DATE), + ("2026-12", "[2026-12-01,2027-01-01)", DATE), + ("June 2026", "[2026-06-01,2026-07-01)", DATE), + # A date or a moment is not: it says when something started and left it open. + ("2026-06-10", "[2026-06-10,)", DATE), + ("Jan 15, 2024", "[2024-01-15,)", DATE), + ("2026-06-10T14:00:00", "[2026-06-10T14:00:00.000000Z,)", INSTANT), + ("2026-06-10T14:00:00Z", "[2026-06-10T14:00:00.000000Z,)", INSTANT), + ("2026-06-10T14:00:00+02:00", "[2026-06-10T12:00:00.000000Z,)", INSTANT), + (" 2026-06-10 ", "[2026-06-10,)", DATE), + ], +) +def test_authored_point_denotes_the_span_its_precision_covers(written, literal, axis): + span = parse_authored_point(written) + + assert span is not None + assert str(span) == literal + assert span.axis is axis + assert span.lower_inclusive is True + + +def test_authored_date_never_acquires_a_time_of_day(): + """A calendar date must not become midnight UTC on the way in. + + Midnight in *which* zone is a question the author never answered, and answering it + for them would make a date query and an instant query disagree about the same note. + """ + span = parse_authored_point("2026-06-10") + + assert span is not None + assert span.axis is DATE + assert span.lower == "2026-06-10" + assert "T" not in span.lower and "Z" not in span.lower + + +def test_authored_naive_timestamp_is_read_as_utc_not_local_time(): + """The two spellings of the same moment produce the same stored bound.""" + naive = parse_authored_point("2026-06-10T14:00:00") + explicit = parse_authored_point("2026-06-10T14:00:00Z") + + assert naive is not None and explicit is not None + assert naive == explicit + assert naive.axis is INSTANT + assert naive.lower == "2026-06-10T14:00:00.000000Z" + + +def test_authored_relative_dates_resolve_at_parse_time(): + """`yesterday` is read against the clock now, and re-read on every index pass. + + That is documented behavior rather than a diagnostic: a file edited by hand keeps + its relative wording, and each pass resolves it fresh. + """ + span = parse_authored_point("yesterday") + + assert span is not None + assert span.axis is DATE + yesterday = datetime.now().date() - timedelta(days=1) + assert span.lower == yesterday.isoformat() + + +# The written vocabulary. These are what the *reader* accepts; the qualifier grammar +# then decides how much of a line it can safely claim (see +# tests/markdown/test_temporal_qualifier.py), which is a narrower question. + + +@pytest.mark.parametrize( + ("written", "literal", "axis"), + [ + # Month names, in the orders English writes them. + ("June 10, 2026", "[2026-06-10,)", DATE), + ("10 June 2026", "[2026-06-10,)", DATE), + # The exact forms entity_parser.parse_date already advertises. + ("Jan 15, 2024", "[2024-01-15,)", DATE), + ("2024-01-15", "[2024-01-15,)", DATE), + # A clock reading moves the point onto the instant axis, read as UTC. + ("2024-01-15 10:00 AM", "[2024-01-15T10:00:00.000000Z,)", INSTANT), + ("2026-06-10 10:00 AM", "[2026-06-10T10:00:00.000000Z,)", INSTANT), + ], +) +def test_written_dates_read_on_the_axis_their_precision_names(written, literal, axis): + """A written date stays a date; adding a clock reading is what makes it an instant. + + `June 10, 2026` must never acquire a time of day -- midnight in which zone is a + question the author never answered -- while `10:00 AM` with no offset is UTC, the + same convention every other naive datetime here follows. + """ + span = parse_authored_point(written) + + assert span is not None + assert str(span) == literal + assert span.axis is axis + + +def test_written_relative_dates_resolve_against_now(): + """dateparser's relative vocabulary is read whole when it is handed a whole phrase.""" + span = parse_authored_point("2 days ago") + + assert span is not None + assert span.axis is DATE + assert span.lower == (datetime.now().date() - timedelta(days=2)).isoformat() + + +@pytest.mark.parametrize( + ("written", "date_order", "expected_lower"), + [ + # Year last: YMD cannot apply, so dateparser falls back to day-first and only + # MDY reads it differently. + ("03/04/2026", "YMD", "2026-04-03"), + ("03/04/2026", "DMY", "2026-04-03"), + ("03/04/2026", "MDY", "2026-03-04"), + # Year first: now YMD and DMY disagree, so the three orders are pinned pairwise + # across the two forms and no setting is left unproven. + ("2026/03/04", "YMD", "2026-03-04"), + ("2026/03/04", "DMY", "2026-04-03"), + ("2026/03/04", "MDY", "2026-03-04"), + ], +) +def test_slash_dates_resolve_by_the_configured_order(written, date_order, expected_lower): + span = parse_authored_point(written, date_order=date_order) + + assert span is not None + assert span.lower == expected_lower + assert span.axis is DATE + + +@pytest.mark.parametrize( + ("date_order", "expected_lower"), + [("YMD", "2026-07-10"), ("DMY", "2026-07-10"), ("MDY", "2026-10-07")], +) +def test_date_order_decides_an_ambiguous_slash_date(date_order, expected_lower): + """`10/07/2026` is July 10 or October 7 depending on the configured preference.""" + span = parse_authored_point("10/07/2026", date_order=date_order) + + assert span is not None + assert span.lower == expected_lower + + +def test_iso_dates_are_never_re_guessed_by_date_order(): + """An ISO date is unambiguous, so no preference may reinterpret it.""" + for date_order in ("YMD", "DMY", "MDY"): + span = parse_authored_point("2026-07-10", date_order=date_order) + assert span is not None + assert span.lower == "2026-07-10", date_order + + +def test_the_default_date_order_is_iso(): + assert DEFAULT_DATE_ORDER == "YMD" + assert parse_authored_point("10/07/2026") == parse_authored_point( + "10/07/2026", date_order="YMD" + ) + + +@pytest.mark.parametrize( + "written", + [ + "2026-02-30", # ISO-shaped, but February has no 30th + "2026-13-01", # ISO-shaped, but there is no 13th month + ], +) +def test_impossible_iso_dates_are_unread_rather_than_re_interpreted(written: str): + """dateparser reads `2026-13-01` as the 13th of January; a wrong date is worse. + + The canonical ISO shape takes the strict path precisely so leniency cannot invent + a date the author did not write. + """ + assert parse_authored_point(written) is None + + +@pytest.mark.parametrize( + "written", + [ + "2026-13-01T10:00:00", # RFC 3339-shaped, but there is no 13th month + "2026-02-30T10:00:00Z", # RFC 3339-shaped, but February has no 30th + "2026-06-10T25:00:00+02:00", # RFC 3339-shaped, but there is no 25th hour + ], +) +def test_impossible_iso_timestamps_are_unread_rather_than_re_interpreted(written: str): + """dateparser reads `2026-13-01T10:00:00` as 10:00 on the 13th of January. + + The canonical timestamp shape takes the same strict path the canonical date shape + does, and for the same reason: an instant nobody wrote would be re-projected by every + reindex, while an unread token merely stays observation content. + """ + assert parse_authored_point(written) is None + + +@pytest.mark.parametrize( + ("written", "lower"), + [ + ("2026-06-10T14:00", "2026-06-10T14:00:00.000000Z"), # no seconds + ("2026-06-10 10:00 AM", "2026-06-10T10:00:00.000000Z"), # written the human way + ], +) +def test_flexible_timestamp_spellings_still_reach_the_lenient_reader(written: str, lower: str): + """Only the *canonical* timestamp shape is held to the strict parser. + + The strict branch above is a shape test, not a ban on clock readings: a spelling the + canonical form does not cover is still the convenient form, and dateparser reads it. + """ + span = parse_authored_point(written) + + assert span is not None + assert span.axis is INSTANT + assert span.lower == lower + + +@pytest.mark.parametrize( + "written", + [ + # A month that does not exist, with nothing after it. The two strict branches + # above match only a token that is *exactly* a canonical date or timestamp, so + # this shape used to reach dateparser untouched. + "2026-13", + "2026-00", + # ...the same, carrying a time the canonical shape does not cover: separated by + # a space rather than `T`, or written to minute precision. + "2026-13-01 10:00:00", + "2026-13-01 10:00", + "2026-13-01T10:00", + "2026-02-30 10:00:00", + "2026-13-01 10:00:00Z", + "2026-06-31T09:30", + ], +) +def test_iso_shaped_points_with_impossible_components_are_unread(written: str): + """An ISO-shaped point must mean its components literally, whatever trails it. + + dateparser reads month 13 as *day* 13 and then supplies the month from today, so + these all used to file a date nobody wrote. Guarding only the two canonical shapes + left every other ISO spelling -- a bare year-month, a space-separated timestamp, a + minute-precision one -- on the lenient path. + """ + assert parse_authored_point(written) is None + + +@pytest.mark.parametrize( + "written", + [ + # The reported shape: a mistyped ISO date whose day ran on into a fourth digit. + # dateparser chopped it back to a bare year-month and filed the whole of January, + # so one slipped keystroke widened a single day into a month-long range. + "2026-01-0100", + # The same slip one component earlier, which filed the whole *year* 2026. + "2026-0100", + # Shorter over-long runs. dateparser already declined to read these, but they are + # the same malformed shape and the guard now owns them rather than trusting it to. + "2026-013", + "2026-06-100", + "2026-01-011", + # An unpadded component is a legitimate spelling, so width alone cannot decide: + # these are refused for their values, exactly as their zero-padded twins are. + "2026-1-99", + "2026-0-5", + # A run long enough to overflow the C long `date` converts to. Refused on width + # before conversion, so the guard reads it as no date rather than raising. + "2026-" + "9" * 40, + ], +) +def test_iso_shaped_points_with_malformed_calendar_runs_are_unread(written: str): + """A mistyped ISO point must stay content, not round off into a plausible range. + + The guard's first cut matched calendar components at a fixed width, so a run of the + wrong width matched *nothing* and fell through to dateparser untouched -- the one + outcome the guard exists to prevent. `2026-01-0100` came back as + `[2026-01-01,2026-02-01)`: a whole month, indistinguishable in the index from a range + the author meant to write. A silently wrong date is worse than an unread token, so a + component too wide to be a month or a day now fails the guard on that basis. + """ + assert parse_authored_point(written) is None + + +@pytest.mark.parametrize( + "written", + [ + # The reported shapes: a calendar date carrying an instant marker with no instant + # behind it. dateparser drops the marker and answers with the bare date, so the + # author reached for a moment and the index recorded a whole open-ended day. + "2026-01-01T", + "2026-01-01Z", + "2026-01-01+14:00", # a real UTC offset -- with no time for it to offset + "2026-01-01-05:00", + # The same defect wearing shapes nobody listed. Naming the marker would have + # caught the three above and missed each of these, which is why the guard asks + # what the reader *returned* rather than what the suffix looks like. + "2026-01-01UTC", + "2026-01-01TZ", + "2026-01-01T ", + "2026-01-01,", + "2026-01-01.", + # A dangling separator, which the guard's previous cut could not even see: its + # trailing `(?![\\d-])` lookahead made the head fail to match, so the token + # skipped the guard entirely and reached the lenient reader. + "2026-01-01-", + "2026-01-01-5", + ], +) +def test_iso_dates_with_a_dangling_instant_suffix_are_unread(written: str): + """A calendar date is a complete point, so only a clock reading may follow one. + + Each of these was peeled off its observation and filed as `[2026-01-01,)` -- a + plausible-looking assertion the author never wrote, re-derived identically by every + reindex. The guard is stated on the whole token rather than on a list of suffixes: + what the reader hands back must account for everything the author typed. + """ + assert parse_authored_point(written) is None + + +@pytest.mark.parametrize( + "written", + [ + # A stray character next to an ISO date makes dateparser abandon the ISO reading + # and re-guess the components under the configured order: June 10 became + # *October 6*. Worse than the dangling markers above, which at least kept the day. + "2026-06-10x", + "2026-06x", + # The same re-guess with a clock reading present, so the reader does come back + # with an instant -- on the wrong date. Only comparing that date against the one + # the author wrote catches it. + "2026-06-10 14:00 x", + "2026-06-10 x 14:00", + # A relative phrase after an absolute date: the reader answers with *today* + # shifted, and the ISO date the author wrote is nowhere in the result. + "2026-06-10 tomorrow 14:00", + "2026-06-10T14:00 yesterday", + ], +) +def test_an_iso_date_whose_suffix_re_guesses_it_is_unread(written: str): + """The reader must come back with the date the author wrote, not a nearby one.""" + assert parse_authored_point(written) is None + + +@pytest.mark.parametrize( + "written", + [ + # The reported shape: one digit more than a canonical instant carries. The lenient + # reader truncated it to `...123456Z`, on the very day the head names, so every + # check the reading makes passed and the index recorded an instant 100ns off the + # one the author wrote -- re-derived identically by every reindex. + "2026-01-01T10:00:00.1234567", + # The same defect wearing every spelling of the syntax around it. None of these is + # distinguishable by what the reader *returned* -- each truncates and each lands on + # the right day -- which is why this one is judged on the text instead. + "2026-01-01t10:00:00.1234567", + "2026-01-01 10:00:00.1234567", + "2026-01-01T10:00:00.1234567Z", + "2026-01-01T10:00:00.1234567z", + "2026-01-01T10:00:00.1234567+02:00", + "2026-01-01T10:00:00.1234567-05:00", + "2026-01-01T10:00:00.1234567+0200", + # Precision far past anything a clock emits, truncated just as quietly: a 20- and a + # 30-digit fraction both stored six digits and discarded the rest without a word. + "2026-01-01T10:00:00.12345678901234567890", + "2026-01-01T10:00:00." + "1" * 30, + ], +) +def test_an_iso_point_finer_than_a_microsecond_is_unread(written: str): + """Over-precision is refused on the lenient path too, not silently rounded. + + `canonical_bound` has always refused these -- dropping digits would store a different + instant than the author wrote -- but that refusal only governed the strict path. A + point one digit too precise never matched `_INSTANT_BOUND`, so it fell to the lenient + reader, which truncated it and answered with a time on the correct day. The day check + is what guards that path, and a truncated fraction sails straight through it: the + digits it drops were never in the answer to be checked. Judged on the author's text + instead, so both readers refuse the same token for the same reason. + """ + assert parse_authored_point(written) is None + + +@pytest.mark.parametrize( + ("written", "lower"), + [ + # Exactly six digits: the widest fraction a canonical instant carries, so it is + # stored whole and nothing is dropped. The refusal above must stop precisely here. + ("2026-01-01T10:00:00.123456", "2026-01-01T10:00:00.123456Z"), + ("2026-01-01 10:00:00.123456", "2026-01-01T10:00:00.123456Z"), + ("2026-01-01T10:00:00.123456Z", "2026-01-01T10:00:00.123456Z"), + ("2026-01-01T10:00:00.123456+02:00", "2026-01-01T08:00:00.123456Z"), + # Narrower fractions were never in question, and are pinned so a future widening + # of the rule cannot quietly take them. + ("2026-01-01T10:00:00.1", "2026-01-01T10:00:00.100000Z"), + ("2026-06-10 14:00:00.5", "2026-06-10T14:00:00.500000Z"), + ], +) +def test_a_fraction_a_canonical_instant_can_hold_still_reads(written: str, lower: str): + """Refusing over-precision must cost nothing that stores losslessly. + + Six digits is the boundary, not "any fraction is suspicious": these name a moment the + canonical form records exactly, so there is no truncation to prevent and no reason to + withhold the assertion. + """ + span = parse_authored_point(written) + + assert span is not None + assert span.axis is INSTANT + assert span.lower == lower + + +@pytest.mark.parametrize("today", ["2026-03-07", "2026-09-01"]) +def test_a_clock_reading_on_a_month_is_not_completed_from_the_indexing_date(today: str): + """`2026-13`'s disease in the suffix: a time of day needs a day to fall on. + + `2026-06 10:00` gave dateparser a year, a month and a clock but no day, and it filled + the day from the current date -- `[2026-06-07T10:00...,)` in March, + `[2026-06-01T10:00...,)` in September. The same note projected different valid time on + different days. A head that names only a month owns no day, so nothing may trail it. + """ + with freeze_time(today): + assert parse_authored_point("2026-06 10:00") is None + + +@pytest.mark.parametrize( + ("written", "lower"), + [ + # The boundary the suffix rule draws is "did the reader turn this into a time on + # that date?", not "does this look like a clock?". These carry no colon and no + # digit at all, yet each really is the time it claims to be, so each still reads. + ("2026-06-10 noon", "2026-06-10T12:00:00.000000Z"), + ("2026-06-10 midnight", "2026-06-10T00:00:00.000000Z"), + ("2026-06-10 2pm", "2026-06-10T14:00:00.000000Z"), + ("2026-06-10 at 14:00", "2026-06-10T14:00:00.000000Z"), + # An offset and a zone are only dangling when there is no time in front of them. + ("2026-06-10T14:00Z", "2026-06-10T14:00:00.000000Z"), + ("2026-06-10 14:00:00.5", "2026-06-10T14:00:00.500000Z"), + ("2026-06-10 14:00:00 UTC", "2026-06-10T14:00:00.000000Z"), + ("2026-06-10T14:00:00+0200", "2026-06-10T12:00:00.000000Z"), + ], +) +def test_a_real_time_of_day_still_follows_an_iso_date(written: str, lower: str): + """Refusing a dangling suffix must not cost a genuine one. + + A rule written as a grammar for what may follow a date would have taken these with + it: none of them is RFC 3339, and half of them do not start with a digit. Deciding on + the reader's answer instead leaves every spelling it can genuinely read. + """ + span = parse_authored_point(written) + + assert span is not None + assert span.axis is INSTANT + assert span.lower == lower + + +@pytest.mark.parametrize( + ("written", "lower"), + [ + # A clock reading on a date written in no machine syntax at all. The ISO rules + # never see these -- there is no literal reading to hold them to -- so the + # flexible reader's answer is taken as given, clock and all. + ("10/07/2026 14:00", "2026-07-10T14:00:00.000000Z"), + ("10/07/2026 14:00:00+02:00", "2026-07-10T12:00:00.000000Z"), + ("June 10, 2026 2pm", "2026-06-10T14:00:00.000000Z"), + ("June 10, 2026 at 14:00", "2026-06-10T14:00:00.000000Z"), + ], +) +def test_a_non_iso_date_may_carry_a_clock_reading(written: str, lower: str): + """Both readers file instants, and only one of them checks the date it was given. + + An ISO head is authoritative, so a clock reading beside one is verified against it. + These spellings have no such head -- `@occurred:"June 10, 2026 2pm"` says everything + it means through the flexible reader -- so nothing here is second-guessed. + """ + span = parse_authored_point(written) + + assert span is not None + assert span.axis is INSTANT + assert span.lower == lower + + +@pytest.mark.parametrize("today", ["2026-03-07", "2026-09-01"]) +def test_an_impossible_iso_month_is_not_completed_from_the_indexing_date(today: str): + """The worst shape of all: a date whose meaning depended on when the reindex ran. + + `2026-13` gave dateparser a year and a day but no month, and it filled the gap from + the current date -- `[2026-03-13,)` in March, `[2026-09-13,)` in September. The same + note projected different valid time on different days, so a query that matched it + last week could stop matching it today with nothing having been edited. + """ + with freeze_time(today): + assert parse_authored_point("2026-13") is None + + +@pytest.mark.parametrize( + ("written", "literal", "axis"), + [ + # A real year-month, which is still read as the month it delimits. + ("2026-06", "[2026-06-01,2026-07-01)", DATE), + ("9999-12", "[9999-12-01,)", DATE), + # A real date carrying a time the canonical `T` shape does not cover. These are + # the spellings the guard above is closest to, so they are pinned explicitly. + ("2026-06-10 14:00:00", "[2026-06-10T14:00:00.000000Z,)", INSTANT), + ("2026-06-10 14:00:00Z", "[2026-06-10T14:00:00.000000Z,)", INSTANT), + ("2026-06-10 14:00:00+02:00", "[2026-06-10T12:00:00.000000Z,)", INSTANT), + ("2026-06-10 10:00", "[2026-06-10T10:00:00.000000Z,)", INSTANT), + ("2026-06-10T14:00", "[2026-06-10T14:00:00.000000Z,)", INSTANT), + ("2026-06-10 10:00 AM", "[2026-06-10T10:00:00.000000Z,)", INSTANT), + # Not ISO-shaped at all: single-digit components, slashes, words, relative + # phrases. The guard must not so much as look at these. + ("2026-1-5", "[2026-01-05,)", DATE), + ("2026/03/04", "[2026-03-04,)", DATE), + ("June 10, 2026", "[2026-06-10,)", DATE), + ], +) +def test_the_iso_guard_leaves_every_readable_spelling_to_the_lenient_reader( + written: str, literal: str, axis +): + """The guard is a validity test on ISO components, not a ban on flexible spellings. + + Refusing an impossible ISO date must cost nothing that already reads. Anything whose + leading components name a real date -- and anything not ISO-shaped at all -- goes on + reaching dateparser exactly as before, so a future tightening cannot quietly take + these spellings without failing here. + """ + span = parse_authored_point(written) + + assert span is not None + assert str(span) == literal + assert span.axis is axis + + +@pytest.mark.parametrize( + "written", + ["paul", "basicmemory.com", "ops@example.com", "someone(2026)", "Redis.", "Q3"], +) +def test_text_that_names_no_date_reads_as_nothing(written: str): + """No error and no assertion: the caller leaves such a token as content.""" + assert parse_authored_point(written) is None + + +@pytest.mark.parametrize( + ("written", "literal"), + [ + # The last year and the last month have no successor to close at, so the + # canonical form for them is unbounded -- exactly as it is for an inclusive + # upper bound on the last date. + ("9999", "[9999-01-01,)"), + ("9999-12", "[9999-12-01,)"), + # The last day was always open-ended, like every other day. + ("9999-12-31", "[9999-12-31,)"), + ], +) +def test_periods_at_the_end_of_the_calendar_run_to_the_end_of_it(written: str, literal: str): + """Unbounded above loses no days: nothing follows 9999-12-31. + + `[9999-12-01,)` holds exactly the days a closed `[9999-12-01,10000-01-01)` would -- + and year 10000 is not a date Python can build. Constructing it raised `ValueError` + straight through `parse_authored_point` and `parse_temporal_qualifier` into the + markdown parser, failing the *whole note* over one qualifier, which is the one thing + the qualifier contract promises can never happen. + """ + span = parse_authored_point(written) + + assert span is not None + assert str(span) == literal + assert span.is_empty is False + + +@pytest.mark.parametrize( + ("written", "literal"), + [("9998", "[9998-01-01,9999-01-01)"), ("9999-11", "[9999-11-01,9999-12-01)")], +) +def test_the_period_before_the_calendar_edge_still_closes(written: str, literal: str): + """The open upper end is the calendar's edge, not "four digits" or "December".""" + span = parse_authored_point(written) + + assert span is not None + assert str(span) == literal + + +def test_a_year_beyond_the_calendar_is_unread(): + """Year 10000 is not a date at all, so the token names nothing and stays content.""" + assert parse_authored_point("10000") is None + + +@pytest.mark.parametrize( + "written", + [ + # The canonical shape, read exactly and refused by the ISO reader... + "9999-12-31T23:59:59-05:00", + # ...the same moment spelled loosely, still ISO-headed, so the ISO reader asks + # the flexible one for the clock and then finds the moment unstorable... + "9999-12-31 23:59:59 -05:00", + # ...and the same moment in no machine syntax at all, which the flexible reader + # owns outright. + "December 31, 9999 23:59:59 -05:00", + ], +) +def test_an_authored_instant_that_leaves_the_calendar_in_utc_is_unread(written: str): + """The flexible reader has no bound to refuse, so it reads no date at all. + + Its contract is None-for-unreadable, not an exception: `parse_temporal_qualifier` + does not guard this call, so anything raised here fails the note. All three spellings + are pinned because they take different routes to the same refusal. + """ + assert parse_authored_point(written) is None + + +# --- TemporalRange normalization --- + + +def test_unbounded_sides_are_forced_exclusive(): + """PostgreSQL's rule: there is no endpoint to include, so inclusivity is meaningless. + + Asserted on the instant axis so this rule is the only one moving: a date range + would also be rewritten to `[)`, which is a separate normalization with its own + tests below. + """ + span = TemporalRange( + axis=INSTANT, + lower=None, + upper="2026-07-27T00:00:00.000000Z", + lower_inclusive=True, + upper_inclusive=True, + ) + + assert span.lower_inclusive is False + assert span.upper_inclusive is True + assert str(span) == "(,2026-07-27T00:00:00.000000Z]" + + +def test_fully_unbounded_range_is_exclusive_on_both_sides(): + span = TemporalRange(axis=DATE, lower_inclusive=True, upper_inclusive=True) + + assert (span.lower_inclusive, span.upper_inclusive) == (False, False) + assert str(span) == "(,)" + + +@pytest.mark.parametrize( + ("lower_inclusive", "upper_inclusive"), + [(True, False), (False, True), (False, False)], +) +def test_degenerate_range_collapses_to_empty(lower_inclusive: bool, upper_inclusive: bool): + """`[a,a)`, `(a,a]`, and `(a,a)` contain no points, so they *are* the empty range.""" + span = TemporalRange( + axis=DATE, + lower="2026-07-27", + upper="2026-07-27", + lower_inclusive=lower_inclusive, + upper_inclusive=upper_inclusive, + ) + + assert span.is_empty is True + assert span.lower is None and span.upper is None + assert str(span) == "empty" + + +def test_closed_single_point_range_is_not_empty(): + """`[a,a]` contains exactly one point, which is a real interval. + + On the date axis that one point is one day, and the canonical form says so by + closing at the following day rather than by owning both endpoints. + """ + span = TemporalRange( + axis=DATE, + lower="2026-07-27", + upper="2026-07-27", + lower_inclusive=True, + upper_inclusive=True, + ) + + assert span.is_empty is False + assert str(span) == "[2026-07-27,2026-07-28)" + + +def test_inverted_range_is_refused(): + with pytest.raises(TemporalQualifierError, match="after upper bound"): + TemporalRange(axis=DATE, lower="2026-08-01", upper="2026-06-10") + + +def test_empty_range_cannot_carry_bounds(): + """Two representations of the same interval would make equality lie.""" + with pytest.raises(TemporalQualifierError, match="carries no bounds"): + TemporalRange(axis=DATE, lower="2026-07-27", is_empty=True) + with pytest.raises(TemporalQualifierError, match="carries no bounds"): + TemporalRange(axis=DATE, is_empty=True, upper_inclusive=True) + + +def test_range_rejects_non_canonical_bounds(): + with pytest.raises(TemporalQualifierError, match="not canonical"): + TemporalRange(axis=INSTANT, lower="2026-07-27T18:42:00Z") + + +def test_empty_constructor_builds_the_empty_range_on_one_axis(): + span = TemporalRange.empty(INSTANT) + + assert (span.axis, span.is_empty, span.lower, span.upper) == (INSTANT, True, None, None) + + +# --- The discrete canonical form --- +# +# Calendar dates are a discrete domain, so every date range is stored half-open, the +# way PostgreSQL canonicalizes `daterange`. Without it the scalar endpoint comparisons +# in `repository.temporal_filters` do not decide membership -- see +# `test_date_ranges_that_share_no_day_do_not_overlap` for the case that proves it. + + +@pytest.mark.parametrize( + ("authored", "canonical"), + [ + # Already half-open: nothing moves. + ("[2026-06-10,2026-07-27)", "[2026-06-10,2026-07-27)"), + # An exclusive lower end starts on the following day. + ("(2026-06-10,2026-07-27)", "[2026-06-11,2026-07-27)"), + # An inclusive upper end closes at the start of the following day. + ("[2026-06-10,2026-07-27]", "[2026-06-10,2026-07-28)"), + ("(2026-06-10,2026-07-27]", "[2026-06-11,2026-07-28)"), + # An unbounded side has no endpoint to move, whichever side it is. + ("[2026-06-10,)", "[2026-06-10,)"), + ("(2026-06-10,)", "[2026-06-11,)"), + ("(,2026-07-27)", "(,2026-07-27)"), + ("(,2026-07-27]", "(,2026-07-28)"), + ("(,)", "(,)"), + # One authored day is one canonical day. + ("[2026-07-27,2026-07-27]", "[2026-07-27,2026-07-28)"), + ], +) +def test_date_ranges_are_stored_half_open(authored: str, canonical: str): + """Whatever the author wrote, the stored date range is `[lower,upper)`.""" + span = parse_range_literal(authored, axis=DATE) + + assert str(span) == canonical + # A bounded lower end is always owned, a bounded upper end never is. + assert span.lower_inclusive is (span.lower is not None) + assert span.upper_inclusive is False + + +def test_the_canonical_date_rendering_is_a_fixed_point(): + """Re-parsing what `__str__` produced yields this same value, not a third form.""" + for authored in ("(2026-06-10,2026-07-27]", "[2026-07-27,2026-07-27]", "(,2026-07-27]"): + span = parse_range_literal(authored, axis=DATE) + + assert parse_range_literal(str(span), axis=DATE) == span, authored + + +@pytest.mark.parametrize( + "literal", + [ + "[2026-07-27,2026-07-27)", # opens and closes on the same day + "(2026-07-27,2026-07-27]", # starts the 28th, ends the 27th + "(2026-07-27,2026-07-27)", + # After the 27th and before the 28th there is no day at all. Read as a + # continuous interval this looks non-empty, which is exactly the confusion + # the discrete canonical form removes. + "(2026-07-27,2026-07-28)", + ], +) +def test_date_ranges_that_admit_no_day_are_the_empty_range(literal: str): + span = parse_range_literal(literal, axis=DATE) + + assert span.is_empty is True + assert str(span) == "empty" + + +def test_an_inclusive_upper_end_on_the_last_date_becomes_unbounded(): + """`9999-12-31` has no successor to close against, and no later day to exclude.""" + span = TemporalRange( + axis=DATE, + lower="2026-06-10", + upper="9999-12-31", + lower_inclusive=True, + upper_inclusive=True, + ) + + assert (span.upper, span.upper_inclusive) == (None, False) + assert str(span) == "[2026-06-10,)" + + +def test_the_last_date_alone_is_still_one_day_not_the_empty_range(): + """`[9999-12-31,9999-12-31]` survives the rewrite that drops its upper end.""" + span = TemporalRange( + axis=DATE, + lower="9999-12-31", + upper="9999-12-31", + lower_inclusive=True, + upper_inclusive=True, + ) + + assert span.is_empty is False + assert str(span) == "[9999-12-31,)" + + +def test_an_exclusive_lower_end_on_the_last_date_is_empty(): + """A range beginning strictly after the last date admits no date at all.""" + span = TemporalRange(axis=DATE, lower="9999-12-31") + + assert span.is_empty is True + assert str(span) == "empty" + + +@pytest.mark.parametrize( + ("literal", "expected"), + [ + ( + "(2026-07-27T18:42:00Z,2026-07-27T19:00:00Z]", + (False, True, "2026-07-27T18:42:00.000000Z", "2026-07-27T19:00:00.000000Z"), + ), + ( + "[2026-07-27T18:42:00Z,2026-07-27T19:00:00Z]", + (True, True, "2026-07-27T18:42:00.000000Z", "2026-07-27T19:00:00.000000Z"), + ), + ("(,2026-07-27T19:00:00Z]", (False, True, None, "2026-07-27T19:00:00.000000Z")), + ], +) +def test_instant_ranges_keep_the_inclusivity_they_were_written_with(literal, expected): + """Instants are continuous: there is no "next instant" to shift a bound onto. + + Adding a microsecond would be an invented precision, and rewriting an instant the + way a date is rewritten would move the endpoint to a moment nobody wrote. + """ + span = parse_range_literal(literal, axis=INSTANT) + + assert (span.lower_inclusive, span.upper_inclusive, span.lower, span.upper) == expected + + +def test_an_instant_range_over_one_day_is_not_widened_by_a_day(): + """The date rewrite must not reach the instant axis: `+1 day` there is a bug.""" + span = parse_range_literal("[2026-07-27T00:00:00Z,2026-07-27T23:59:59Z]", axis=INSTANT) + + assert span.upper == "2026-07-27T23:59:59.000000Z" + assert span.upper_inclusive is True + + +def test_a_degenerate_instant_range_still_holds_exactly_one_moment(): + """`[t,t]` on a continuous axis stays `[t,t]`; there is no successor to close at.""" + span = parse_range_literal("[2026-07-27T18:42:00Z,2026-07-27T18:42:00Z]", axis=INSTANT) + + assert span.is_empty is False + assert str(span) == "[2026-07-27T18:42:00.000000Z,2026-07-27T18:42:00.000000Z]" + + +# --- Range literals --- + + +@pytest.mark.parametrize( + ("literal", "expected"), + [ + # Date literals already in the canonical half-open form. + ("[2026-06-10,2026-07-27)", (True, False, "2026-06-10", "2026-07-27")), + ("[2026-06-10,)", (True, False, "2026-06-10", None)), + ("(,2026-07-27)", (False, False, None, "2026-07-27")), + # Instant literals, which are stored exactly as written whatever the brackets. + ( + "(2026-06-10T00:00:00.000000Z,2026-07-27T00:00:00.000000Z]", + (False, True, "2026-06-10T00:00:00.000000Z", "2026-07-27T00:00:00.000000Z"), + ), + ( + "[2026-06-10T00:00:00.000000Z,2026-07-27T00:00:00.000000Z]", + (True, True, "2026-06-10T00:00:00.000000Z", "2026-07-27T00:00:00.000000Z"), + ), + ("(,2026-07-27T00:00:00.000000Z]", (False, True, None, "2026-07-27T00:00:00.000000Z")), + ], +) +def test_range_literal_round_trips_through_its_canonical_rendering(literal, expected): + """A literal already in canonical form parses and renders back to itself. + + Date literals written some other way still round trip -- through their canonical + spelling rather than their authored one -- which + `test_the_canonical_date_rendering_is_a_fixed_point` pins separately. + """ + span = parse_range_literal(literal) + + assert (span.lower_inclusive, span.upper_inclusive, span.lower, span.upper) == expected + assert str(span) == literal + + +def test_range_literal_tolerates_surrounding_whitespace(): + assert str(parse_range_literal(" [2026-06-10, 2026-07-27) ")) == "[2026-06-10,2026-07-27)" + + +def test_empty_literal_requires_an_explicit_axis(): + """`empty` carries no bounds to classify, so the caller must name the axis.""" + assert parse_range_literal("empty", axis=DATE).is_empty is True + with pytest.raises(TemporalQualifierError, match="axis must be given"): + parse_range_literal("empty") + + +def test_fully_unbounded_literal_requires_an_explicit_axis(): + assert parse_range_literal("(,)", axis=INSTANT).axis is INSTANT + with pytest.raises(TemporalQualifierError, match="no bounds to classify"): + parse_range_literal("(,)") + + +def test_range_literal_refuses_mixed_axes(): + with pytest.raises(TemporalQualifierError, match="mix date-only and timestamp bounds"): + parse_range_literal("[2026-06-10,2026-07-27T00:00:00Z)") + + +def test_range_literal_refuses_an_axis_it_was_not_asked_for(): + with pytest.raises(TemporalQualifierError, match="expected instant bounds"): + parse_range_literal("[2026-06-10,2026-07-27)", axis=INSTANT) + + +@pytest.mark.parametrize( + "literal", + [ + "2026-06-10,2026-07-27", # no brackets + "[2026-06-10]", # no comma + "[2026-06-10,2026-07-27", # unbalanced + "[2026-06-10,2026-07-27,2026-08-01)", # three bounds + "", + ], +) +def test_malformed_range_literals_are_refused(literal: str): + with pytest.raises(TemporalQualifierError, match="range literal must be"): + parse_range_literal(literal) + + +# --- TemporalFilter --- + + +def test_filter_refuses_asking_two_questions_at_once(): + with pytest.raises(TemporalQualifierError, match="never both"): + TemporalFilter( + at=parse_point("2026-07-27"), + overlaps=parse_range_literal("[2026-06-10,2026-07-27)"), + ) + + +def test_filter_refuses_asking_nothing_at_all(): + """A filter that names no kind, point, or range would match everything silently.""" + with pytest.raises(TemporalQualifierError, match="must name a kind"): + TemporalFilter() + + +def test_point_filter_window_is_the_degenerate_closed_range(): + """Containment is overlap with `[p,p]`, which is why one predicate answers both.""" + window = TemporalFilter(at=parse_point("2026-07-27")).window + + assert window == TemporalRange( + axis=DATE, + lower="2026-07-27", + upper="2026-07-27", + lower_inclusive=True, + upper_inclusive=True, + ) + # Canonicalized like any other date range: still the single day 2026-07-27, now in + # the half-open form the SQL predicate compares correctly. + assert str(window) == "[2026-07-27,2026-07-28)" + + +def test_instant_point_filter_window_stays_a_closed_moment(): + """The instant axis has no successor to close at, so `[t,t]` is the window.""" + window = TemporalFilter(at=parse_point("2026-07-27T18:42:00Z")).window + + assert str(window) == "[2026-07-27T18:42:00.000000Z,2026-07-27T18:42:00.000000Z]" + + +def test_overlap_filter_window_is_the_range_itself(): + span = parse_range_literal("[2026-06-10,2026-07-27)") + + assert TemporalFilter(overlaps=span).window == span + + +def test_kind_only_filter_has_no_window(): + """Nothing to intersect: the question is only "does this axis carry an assertion".""" + assert TemporalFilter(kind=TimeKind.EFFECTIVE).window is None + + +# --- TemporalAssertion --- + + +def test_assertion_defaults_to_the_observation_extractor(): + assertion = TemporalAssertion( + time_kind=TimeKind.EFFECTIVE, + valid_during=parse_range_literal("[2026-06-10,2026-07-27)"), + source_text="@effective[2026-06-10,2026-07-27)", + ) + + assert assertion.extractor == "observation" + assert assertion.metadata is None + + +def test_recorded_time_is_not_an_authorable_kind(): + """Recorded time is never written in markdown, so no kind names it.""" + assert "recorded" not in {kind.value for kind in TimeKind} + assert {kind.value for kind in TimeKind} == { + "effective", + "valid", + "occurred", + "due", + "mentioned", + }