fix vector search implementation

jordan-smith721 · jordan-smith721 · commit dbb1dbf8cd57 · 2025-11-06T12:42:09.000-05:00
diff --git a/client/app/lib/constants.ts b/client/app/lib/constants.ts
@@ -7,6 +7,7 @@ export const APP_CONFIG = {
   description: 'Browse movies from the sample MFlix database',
   defaultMovieLimit: 20,
   maxMovieLimit: 100,
+  vectorSearchPageSize: 20, // Fixed page size for vector search results display
   imageFormats: ['image/avif', 'image/webp'],
 } as const;
 
diff --git a/client/app/movies/page.tsx b/client/app/movies/page.tsx
@@ -239,16 +239,18 @@ export default function Movies() {
       let result;
 
       if (searchParams.searchType === 'vector-search') {
-        // Vector Search: Fetch all results and implement client-side pagination
+        // Vector Search: Use the limit from search params as the fetch limit
         const vectorSearchParams = {
           q: searchParams.q!,
-          limit: searchParams.limit || 50, // Get more results for better pagination experience
+          limit: searchParams.limit || 50, // This is how many results to fetch from backend
         };
+        
         result = await vectorSearchMovies(vectorSearchParams);
         
         if (result.success) {
           const allResults = result.movies || [];
-          const pageSize = searchParams.limit || 20;
+          const pageSize = APP_CONFIG.vectorSearchPageSize; // Fixed page size for UI display
+          
           setAllVectorSearchResults(allResults);
           setVectorSearchPage(1);
           setVectorSearchPageSize(pageSize);
@@ -257,10 +259,7 @@ export default function Movies() {
           const firstPageResults = allResults.slice(0, pageSize);
           setSearchResults(firstPageResults);
           
-          // Set pagination state based on total results
-          const totalPages = Math.ceil(allResults.length / pageSize);
-          setSearchHasNextPage(totalPages > 1);
-          setSearchHasPrevPage(false);
+          // Set pagination state for vector search
           setSearchTotalCount(allResults.length);
         }
       } else {
@@ -352,15 +351,19 @@ export default function Movies() {
       return { paginatedResults: [], hasNext: false, hasPrev: false, totalPages: 0 };
     }
 
+    const totalResults = allVectorSearchResults.length;
+    const totalPages = Math.ceil(totalResults / vectorSearchPageSize);
+    const hasNext = vectorSearchPage < totalPages;
+    const hasPrev = vectorSearchPage > 1;
+    
     const startIndex = (vectorSearchPage - 1) * vectorSearchPageSize;
     const endIndex = startIndex + vectorSearchPageSize;
     const paginatedResults = allVectorSearchResults.slice(startIndex, endIndex);
-    const totalPages = Math.ceil(allVectorSearchResults.length / vectorSearchPageSize);
     
     return {
       paginatedResults,
-      hasNext: vectorSearchPage < totalPages,
-      hasPrev: vectorSearchPage > 1,
+      hasNext,
+      hasPrev,
       totalPages
     };
   };
@@ -653,55 +656,60 @@ export default function Movies() {
                     /* Vector search results with client-side pagination */
                     (() => {
                       const { hasNext, hasPrev, totalPages } = getVectorSearchPageData();
-                      return totalPages > 1 ? (
-                        <nav className={movieStyles.pagination} aria-label="Vector search results pagination">
-                          <div className={movieStyles.paginationContainer}>
-                            {/* Previous Button */}
-                            {hasPrev ? (
-                              <button
-                                onClick={() => handleVectorSearchPageChange(vectorSearchPage - 1)}
-                                className={movieStyles.pageButton}
-                                aria-label="Go to previous page"
-                              >
-                                ← Previous
-                              </button>
-                            ) : (
-                              <span className={`${movieStyles.pageButton} ${movieStyles.disabled}`}>
-                                ← Previous
-                              </span>
-                            )}
-
-                            {/* Current Page Info */}
-                            <div className={movieStyles.pageInfo}>
-                              Page {vectorSearchPage} of {totalPages}
+                      
+                      if (totalPages > 1) {
+                        return (
+                          <nav className={movieStyles.pagination} aria-label="Vector search results pagination">
+                            <div className={movieStyles.paginationContainer}>
+                              {/* Previous Button */}
+                              {hasPrev ? (
+                                <button
+                                  onClick={() => handleVectorSearchPageChange(vectorSearchPage - 1)}
+                                  className={movieStyles.pageButton}
+                                  aria-label="Go to previous page"
+                                >
+                                  ← Previous
+                                </button>
+                              ) : (
+                                <span className={`${movieStyles.pageButton} ${movieStyles.disabled}`}>
+                                  ← Previous
+                                </span>
+                              )}
+
+                              {/* Current Page Info */}
+                              <div className={movieStyles.pageInfo}>
+                                Page {vectorSearchPage} of {totalPages}
+                              </div>
+
+                              {/* Next Button */}
+                              {hasNext ? (
+                                <button
+                                  onClick={() => handleVectorSearchPageChange(vectorSearchPage + 1)}
+                                  className={movieStyles.pageButton}
+                                  aria-label="Go to next page"
+                                >
+                                  Next →
+                                </button>
+                              ) : (
+                                <span className={`${movieStyles.pageButton} ${movieStyles.disabled}`}>
+                                  Next →
+                                </span>
+                              )}
                             </div>
 
-                            {/* Next Button */}
-                            {hasNext ? (
-                              <button
-                                onClick={() => handleVectorSearchPageChange(vectorSearchPage + 1)}
-                                className={movieStyles.pageButton}
-                                aria-label="Go to next page"
-                              >
-                                Next →
-                              </button>
-                            ) : (
-                              <span className={`${movieStyles.pageButton} ${movieStyles.disabled}`}>
-                                Next →
-                              </span>
-                            )}
-                          </div>
-
-                          {/* Additional Info */}
-                          <div className={movieStyles.additionalInfo}>
-                            {vectorSearchPageSize} movies per page • {allVectorSearchResults.length} total results
+                            {/* Additional Info */}
+                            <div className={movieStyles.additionalInfo}>
+                              {vectorSearchPageSize} movies per page • {allVectorSearchResults.length} total results
+                            </div>
+                          </nav>
+                        );
+                      } else {
+                        return (
+                          <div className={movieStyles.searchInfo}>
+                            Showing {allVectorSearchResults.length} results (vector search)
                           </div>
-                        </nav>
-                      ) : (
-                        <div className={movieStyles.searchInfo}>
-                          Showing {allVectorSearchResults.length} results (vector search)
-                        </div>
-                      );
+                        );
+                      }
                     })()
                   )
                 ) : (
diff --git a/server/express/src/controllers/movieController.ts b/server/express/src/controllers/movieController.ts
@@ -726,11 +726,11 @@ export async function vectorSearchMovies(req: Request, res: Response): Promise<v
     // Generate embedding using Voyage AI REST API
     const queryVector = await generateVoyageEmbedding(q.trim(), process.env.VOYAGE_API_KEY);
 
-    // Get embedded movies collection
+    // Get embedded movies collection for vector search
     const embeddedMoviesCollection = getCollection("embedded_movies");
 
-    // Build the $vectorSearch aggregation pipeline
-    const pipeline = [
+    // Step 1: Build the $vectorSearch aggregation pipeline for embedded_movies
+    const vectorSearchPipeline = [
       {
         $vectorSearch: {
           index: "vector_index",
@@ -743,27 +743,94 @@ export async function vectorSearchMovies(req: Request, res: Response): Promise<v
       {
         $project: {
           _id: 1,
-          title: 1,
-          plot: 1,
           score: { $meta: "vectorSearchScore" },
         },
       },
     ];
 
-    const results = await embeddedMoviesCollection.aggregate(pipeline).toArray();
+    // Execute vector search to get movie IDs and scores
+    const vectorResults = await embeddedMoviesCollection.aggregate(vectorSearchPipeline).toArray();
+
+    if (vectorResults.length === 0) {
+      res.json(
+        createSuccessResponse(
+          [],
+          `No similar movies found for query: '${q}'`
+        )
+      );
+      return;
+    }
+
+    // Extract movie IDs and create score mapping
+    const movieIds = vectorResults.map(doc => doc._id);
+    const scoreMap = new Map();
+    vectorResults.forEach(doc => {
+      scoreMap.set(doc._id.toString(), doc.score);
+    });
+
+    // Step 2: Fetch complete movie data from the movies collection
+    const moviesCollection = getCollection("movies");
+    
+    // Build aggregation pipeline to safely handle year field and get complete movie data
+    const moviesPipeline = [
+      {
+        $match: {
+          _id: { $in: movieIds }
+        }
+      },
+      {
+        $project: {
+          _id: 1,
+          title: 1,
+          plot: 1,
+          poster: 1,
+          genres: 1,
+          directors: 1,
+          cast: 1,
+          // Safely convert year to integer, handling strings and dirty data
+          year: {
+            $cond: {
+              if: {
+                $and: [
+                  { $ne: ["$year", null] },
+                  { $eq: [{ $type: "$year" }, "int"] }
+                ]
+              },
+              then: "$year",
+              else: null
+            }
+          }
+        }
+      }
+    ];
+
+    const movieResults = await moviesCollection.aggregate(moviesPipeline).toArray();
+
+    // Step 3: Combine movie data with vector search scores
+    const finalResults: VectorSearchResult[] = movieResults.map(movie => {
+      const movieIdStr = movie._id.toString();
+      const score = scoreMap.get(movieIdStr) || 0;
+
+      return {
+        _id: movieIdStr,
+        title: movie.title || '',
+        plot: movie.plot,
+        poster: movie.poster,
+        year: movie.year,
+        genres: movie.genres || [],
+        directors: movie.directors || [],
+        cast: movie.cast || [],
+        score: score,
+      };
+    });
 
-    // Convert ObjectId to string for each result
-    const vectorResults: VectorSearchResult[] = results.map((doc) => ({
-      _id: doc._id.toString(),
-      title: doc.title,
-      plot: doc.plot,
-      score: doc.score,
-    }));
+    // Sort results by score (highest first) to maintain relevance order
+    finalResults.sort((a, b) => b.score - a.score);
 
     res.json(
       createSuccessResponse(
-        vectorResults,
-        `Found ${vectorResults.length} similar movies for query: '${q}'`
+        finalResults,
+        `Found ${finalResults.length} similar movies for query: '${q}'`
       )
     );
   } catch (error) {
diff --git a/server/express/src/types/index.ts b/server/express/src/types/index.ts
@@ -193,6 +193,11 @@ export interface VectorSearchResult {
   _id: string;
   title: string;
   plot?: string;
+  poster?: string;
+  year?: number;
+  genres?: string[];
+  directors?: string[];
+  cast?: string[];
   score: number;
 }
 
diff --git a/server/express/tests/controllers/movieController.test.ts b/server/express/tests/controllers/movieController.test.ts
@@ -15,6 +15,7 @@ import {
   SAMPLE_MOVIES,
   SAMPLE_SEARCH_RESULTS,
   SAMPLE_VECTOR_RESULTS,
+  SAMPLE_VECTOR_MOVIES,
   SAMPLE_COMMENTS_AGGREGATION,
   SAMPLE_YEARS_AGGREGATION,
   SAMPLE_DIRECTORS_AGGREGATION,
@@ -785,8 +786,17 @@ describe("Movie Controller Tests", () => {
         json: () => Promise.resolve(MOCK_VOYAGE_RESPONSE),
       } as any);
 
-      // Mock database response
-      mockToArray.mockResolvedValue(SAMPLE_VECTOR_RESULTS);
+      // Ensure SAMPLE_VECTOR_RESULTS and SAMPLE_VECTOR_MOVIES have matching IDs
+      const vectorResultsWithMatchingIds = SAMPLE_VECTOR_RESULTS.map((result, index) => ({
+        ...result,
+        _id: SAMPLE_VECTOR_MOVIES[index]._id,
+      }));
+
+      // Mock database responses - first call for embedded_movies collection (vector search)
+      mockToArray
+        .mockResolvedValueOnce(vectorResultsWithMatchingIds)
+        // Second call for movies collection (complete movie data)
+        .mockResolvedValueOnce(SAMPLE_VECTOR_MOVIES);
 
       mockRequest.query = { q: "space adventure", limit: "3" };
 
@@ -803,14 +813,22 @@ describe("Movie Controller Tests", () => {
         })
       );
 
+      // Should call embedded_movies collection first, then movies collection
       expect(mockGetCollection).toHaveBeenCalledWith("embedded_movies");
-      expect(mockAggregate).toHaveBeenCalled();
-
-      const expectedResults = SAMPLE_VECTOR_RESULTS.map((result) => ({
-        _id: result._id.toString(),
-        title: result.title,
-        plot: result.plot,
-        score: result.score,
+      expect(mockGetCollection).toHaveBeenCalledWith("movies");
+      expect(mockAggregate).toHaveBeenCalledTimes(2);
+
+      // Verify the final result structure includes all movie fields
+      const expectedResults = SAMPLE_VECTOR_MOVIES.map((movie, index) => ({
+        _id: movie._id.toString(),
+        title: movie.title,
+        plot: movie.plot,
+        poster: movie.poster,
+        year: movie.year,
+        genres: movie.genres,
+        directors: movie.directors,
+        cast: movie.cast,
+        score: SAMPLE_VECTOR_RESULTS[index].score,
       }));
 
       expect(mockCreateSuccessResponse).toHaveBeenCalledWith(
@@ -888,20 +906,28 @@ describe("Movie Controller Tests", () => {
         json: () => Promise.resolve(MOCK_VOYAGE_RESPONSE),
       } as any);
 
-      mockToArray.mockResolvedValue([]);
+      // Mock empty results from vector search
+      mockToArray
+        .mockResolvedValueOnce([]) // empty vector search results
+        .mockResolvedValueOnce([]); // empty movie results
 
       mockRequest.query = { q: "test" };
 
       await vectorSearchMovies(mockRequest as Request, mockResponse as Response);
 
       expect(mockCreateSuccessResponse).toHaveBeenCalledWith(
         [],
-        "Found 0 similar movies for query: 'test'"
+        "No similar movies found for query: 'test'"
       );
     });
   });
 
   describe("findSimilarMovies", () => {
+    beforeEach(() => {
+      // Reset mocks specifically for findSimilarMovies tests
+      mockToArray.mockReset();
+    });
+
     it("should successfully find similar movies", async () => {
       const targetMovie = {
         _id: new ObjectId(TEST_MOVIE_ID),
diff --git a/server/express/tests/utils/testHelpers.ts b/server/express/tests/utils/testHelpers.ts
