Improve search: fix tag info, use ILIKE, add FTS, increase limit to 50

- Fix incorrect tag coverage info (was "3 posts", actually 3,362 posts / 88%)
- Replace word boundary regex with ILIKE for flexible matching
(now "Russia" finds "Russian", "Russia's", etc.)
- Add FTS query guidance with ts_rank() for relevance ranking
- Remove non-existent 'dead' column references from schema
- Increase default result limit from 9 to 50 for better coverage
- Update all SQL examples to use ILIKE patterns

Database migrations applied:
- add_fts_to_posts: Added FTS tsvector column with GIN index
- add_trigram_indexes: Enabled pg_trgm extension for fuzzy matching

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

src/query_intent_classifier.py

@@ -165,30 +165,32 @@ class IntentClassifier:
         tags = classification["tags"]
 
         if intent == QueryIntent.KEYWORD:
+            keyword_example = keywords[0] if keywords else "keyword"
             return f"""
 Search using KEYWORD approach:
 - Search terms: {', '.join(keywords)}
 - Search in: posts.title, posts.author, providers.name
-- Use ILIKE with wildcards for flexible matching
-- Do not filter by tags (most posts are not tagged yet)
+- Use LOWER(column) LIKE '%keyword%' for flexible matching
+- Example: LOWER(p.title) LIKE '%{keyword_example}%'
+- This matches word variants: '{keyword_example}', '{keyword_example}n', '{keyword_example}\\'s', etc.
 """
 
         elif intent == QueryIntent.TAG:
             return f"""
 Search using TAG approach:
 - Tag names: {', '.join(tags)}
-- Use LOWER() for case-insensitive matching
+- 88% of posts (3,362) have tags - tag search is highly effective!
+- Use LOWER(t.name) LIKE '%tagname%' for flexible matching
 - Join with post_tags and tags tables
-- Note: Only a few posts are tagged, results may be limited
 """
 
         else:  # HYBRID
             return f"""
-Search using HYBRID approach:
-- Try tags first: {', '.join(tags)}
-- Fall back to keywords: {', '.join(keywords)}
+Search using HYBRID approach (RECOMMENDED):
+- Tags to search: {', '.join(tags)}
+- Keywords to search: {', '.join(keywords)}
 - Use OR logic: tag matches OR keyword matches in title/author
-- This maximizes results since most posts are not tagged yet
+- 88% of posts have tags, so tag search is primary
 
 Recommended SQL pattern:
 SELECT DISTINCT p.id, p.title, p.source_url, p.author, p.published_date, p.image_url, p.type
@@ -196,13 +198,13 @@ FROM posts p
 LEFT JOIN post_tags pt ON p.id = pt.post_id
 LEFT JOIN tags t ON pt.tag_id = t.id
 LEFT JOIN providers pr ON p.provider_id = pr.id
-WHERE
-    LOWER(t.name) = ANY(ARRAY[{', '.join(f"'{tag}'" for tag in tags)}])
-    OR LOWER(p.title) LIKE ANY(ARRAY[{', '.join(f"'%{kw}%'" for kw in keywords)}])
-    OR LOWER(p.author) LIKE ANY(ARRAY[{', '.join(f"'%{kw}%'" for kw in keywords)}])
-    OR LOWER(pr.name) LIKE ANY(ARRAY[{', '.join(f"'%{kw}%'" for kw in keywords)}])
+WHERE (
+    {' OR '.join(f"LOWER(t.name) LIKE '%{tag}%'" for tag in tags)}
+    OR {' OR '.join(f"LOWER(p.title) LIKE '%{kw}%'" for kw in keywords)}
+    OR {' OR '.join(f"LOWER(p.author) LIKE '%{kw}%'" for kw in keywords)}
+)
 ORDER BY p.published_date DESC NULLS LAST
-LIMIT 9
+LIMIT 50
 """
 
 

src/vanna.py

@@ -68,7 +68,7 @@ class CustomSQLSystemPromptBuilder(SystemPromptBuilder):
             prompt += (
                 "\n## Database Schema\n"
                 "Tables:\n"
-                "- posts (id, title, source_url, author, published_date, image_url, type, provider_id, created_at, updated_at, dead)\n"
+                "- posts (id, title, source_url, author, published_date, image_url, type, provider_id, created_at, updated_at, content_markdown, fts)\n"
                 "- providers (id, name)\n"
                 "- provider_attributes (id, provider_id, type, name)\n"
                 "- post_provider_attributes (post_id, attribute_id)\n"
@@ -96,7 +96,7 @@ class CustomSQLSystemPromptBuilder(SystemPromptBuilder):
             "- `providers.name`: name of the publishing organization (e.g., 'Nuanced', 'SND').\n"
             "- `tags.name`: thematic keyword or topic (e.g., '3D', 'AI', 'Design').\n"
             "- `post_tags.weight`: relevance score between a post and a tag.\n"
-            "- `posts.dead`: boolean flag indicating if the post is dead/removed (true = dead, false = active).\n"
+            "- `posts.fts`: tsvector column for full-text search (auto-generated from title and author).\n"
         )
 
         # ======================
@@ -104,33 +104,38 @@ class CustomSQLSystemPromptBuilder(SystemPromptBuilder):
         # ======================
         prompt += (
             "\n## Business Logic\n"
-            "- **ALWAYS filter out dead posts**: Include `WHERE p.dead = false` (or `AND p.dead = false`) in every query. Never return posts where dead = true.\n"
             "- A query mentioning an organization (e.g., 'New York Times') should search both `posts.author` and `providers.name`.\n"
             "- Return all post types (spotlight, resource, insight) unless the user specifies otherwise.\n"
             "- Tags link posts to specific themes or disciplines.\n"
             "- A single post may have multiple tags, awards, or categories.\n"
             "- If the user mentions a year (e.g., 'in 2021'), filter with `EXTRACT(YEAR FROM published_date) = 2021`.\n"
             "- If the user says 'recently', filter posts from the last 90 days.\n"
-            "- Always limit exploratory results to 9 rows.\n"
+            "- Default limit is 50 rows for search results. Use OFFSET for pagination if needed.\n"
             "\n"
-            "## CRITICAL: Search Strategy\n"
-            "**IMPORTANT**: Only 3 posts currently have tags. Most posts (7,245+) are NOT tagged yet.\n"
+            "## Search Strategy\n"
+            "**TAG COVERAGE**: 3,362 posts (88%) have tags. Tag-based search is highly effective!\n"
+            "- 9,105 tags available including countries (russia, china, usa), topics (climate change, politics), and formats (interactive, dataviz)\n"
+            "- Use tag matching as PRIMARY search for topic-based queries\n"
             "\n"
             "**Hybrid Search Approach (RECOMMENDED)**:\n"
-            "- ALWAYS use a hybrid approach combining tag search AND keyword search with OR logic.\n"
-            "- Use LEFT JOINs for tags (not INNER JOIN) so untagged posts are included.\n"
+            "- Combine tag search AND keyword search with OR logic for maximum coverage\n"
+            "- Use LEFT JOINs for tags to also include the 12% of untagged posts\n"
             "\n"
-            "**Keyword Matching - Use PostgreSQL Regex for Exact Word Boundaries**:\n"
-            "- Use ~* operator for case-insensitive regex matching\n"
-            "- Use \\m and \\M for word boundaries (start and end of word)\n"
-            "- Pattern: column ~* '\\\\mkeyword\\\\M'\n"
-            "- Example: p.title ~* '\\\\mf1\\\\M' matches 'F1' but NOT 'profile' or 'if'\n"
-            "- This ensures exact word matching, not substring matching\n"
+            "**Keyword Matching - Use ILIKE for Flexible Matching**:\n"
+            "- Use LOWER(column) LIKE '%keyword%' for case-insensitive substring matching\n"
+            "- Example: LOWER(p.title) LIKE '%russia%' matches 'Russia', 'Russian', 'Russia\\'s', etc.\n"
+            "- This ensures word variants are captured (much better than exact word boundary matching)\n"
+            "- For multi-word searches: LOWER(p.title) LIKE '%new york%'\n"
             "\n"
-            "**When to use tag-only search**: Only if user explicitly mentions 'tagged with' or 'tag:'.\n"
-            "**When to use keyword-only search**: For author/organization names, or when tags are not relevant.\n"
+            "**Full-Text Search (for relevance ranking)**:\n"
+            "- The posts table has an 'fts' column (tsvector) for full-text search\n"
+            "- Use: p.fts @@ plainto_tsquery('english', 'search terms')\n"
+            "- For relevance-ranked results: ORDER BY ts_rank(p.fts, plainto_tsquery('english', 'search terms')) DESC\n"
+            "- FTS handles stemming automatically: 'visualization' matches 'visualizations'\n"
+            "- Combine FTS with ILIKE fallback: WHERE p.fts @@ query OR LOWER(p.title) LIKE '%keyword%'\n"
             "\n"
-            "This ensures maximum result coverage while the database is being enriched with tags.\n"
+            "**When to use tag-only search**: Only if user explicitly mentions 'tagged with' or 'tag:'.\n"
+            "**When to use keyword-only search**: For author/organization names.\n"
         )
 
         # ======================
@@ -167,24 +172,32 @@ class CustomSQLSystemPromptBuilder(SystemPromptBuilder):
             "LEFT JOIN post_tags pt ON p.id = pt.post_id "
             "LEFT JOIN tags t ON pt.tag_id = t.id "
             "LEFT JOIN providers pr ON p.provider_id = pr.id "
-            "WHERE p.dead = false AND (t.name ~* '\\\\mf1\\\\M' OR t.name ~* '\\\\mformula\\\\M' "
-            "OR p.title ~* '\\\\mf1\\\\M' OR p.title ~* '\\\\mformula\\\\M' "
-            "OR p.author ~* '\\\\mf1\\\\M') "
-            "ORDER BY p.published_date DESC NULLS LAST LIMIT 9;\"]\n"
+            "WHERE (LOWER(t.name) LIKE '%f1%' OR LOWER(t.name) LIKE '%formula%' "
+            "OR LOWER(p.title) LIKE '%f1%' OR LOWER(p.title) LIKE '%formula%' "
+            "OR LOWER(p.author) LIKE '%f1%') "
+            "ORDER BY p.published_date DESC NULLS LAST LIMIT 50;\"]\n"
             "\nUser: 'Show me posts from The New York Times'\n"
             "Assistant: [call run_sql with \"SELECT DISTINCT p.id, p.title, p.source_url, p.author, p.published_date, p.image_url, p.type "
             "FROM posts p "
             "LEFT JOIN providers pr ON p.provider_id = pr.id "
-            "WHERE p.dead = false AND (p.author ~* '\\\\mnew\\\\M.*\\\\myork\\\\M.*\\\\mtimes\\\\M' OR pr.name ~* '\\\\mnew\\\\M.*\\\\myork\\\\M.*\\\\mtimes\\\\M') "
-            "ORDER BY p.published_date DESC NULLS LAST LIMIT 9;\"]\n"
+            "WHERE (LOWER(p.author) LIKE '%new york times%' OR LOWER(pr.name) LIKE '%new york times%') "
+            "ORDER BY p.published_date DESC NULLS LAST LIMIT 50;\"]\n"
+            "\nUser: 'Russia' or 'Show me Russia content'\n"
+            "Assistant: [call run_sql with \"SELECT DISTINCT p.id, p.title, p.source_url, p.author, p.published_date, p.image_url, p.type "
+            "FROM posts p "
+            "LEFT JOIN post_tags pt ON p.id = pt.post_id "
+            "LEFT JOIN tags t ON pt.tag_id = t.id "
+            "WHERE (LOWER(t.name) LIKE '%russia%' "
+            "OR LOWER(p.title) LIKE '%russia%' OR LOWER(p.author) LIKE '%russia%') "
+            "ORDER BY p.published_date DESC NULLS LAST LIMIT 50;\"]\n"
             "\nUser: 'interactive visualizations'\n"
             "Assistant: [call run_sql with \"SELECT DISTINCT p.id, p.title, p.source_url, p.author, p.published_date, p.image_url, p.type "
             "FROM posts p "
             "LEFT JOIN post_tags pt ON p.id = pt.post_id "
             "LEFT JOIN tags t ON pt.tag_id = t.id "
-            "WHERE p.dead = false AND (t.name ~* '\\\\minteractive\\\\M' OR p.title ~* '\\\\minteractive\\\\M' "
-            "OR p.title ~* '\\\\mvisualization\\\\M' OR t.name ~* '\\\\mdataviz\\\\M') "
-            "ORDER BY p.published_date DESC NULLS LAST LIMIT 9;\"]\n"
+            "WHERE (LOWER(t.name) LIKE '%interactive%' OR LOWER(p.title) LIKE '%interactive%' "
+            "OR LOWER(p.title) LIKE '%visualization%' OR LOWER(t.name) LIKE '%dataviz%') "
+            "ORDER BY p.published_date DESC NULLS LAST LIMIT 50;\"]\n"
         )
 
         # ======================