fix the requests to the FMP API

src/api/insiders/insider_trading_aggregator.py

@@ -111,68 +111,75 @@ class InsiderTradingAggregator:
 
     async def get_fmp_insider_trades(self, session: aiohttp.ClientSession,
                                      symbol: str, limit: int = 100, filter_days: int = 30) -> List[InsiderTrade]:
-        """Get insider trades from Financial Modeling Prep API"""
+        """
+        Get insider trades from Financial Modeling Prep API
+        Get insider trades from FMP by iterating day-by-day for a specific period.
+        NOTE: This method is inefficient and makes many API calls.
+        """
+        if filter_days > 14:
+            logger.warning(f"FMP date range capped at 14 days. Reducing from {filter_days} to 14.")
+            filter_days = 14
+
         all_trades = []
-        page = 0
-
-        start_date = datetime.now() - timedelta(days=filter_days)
-
-        # IMPROVEMENT: Loop through pages to fetch all available data.
-        keep_fetching = True
-        while keep_fetching:
-            endpoint = "insider-trading"
-            params = {'symbol': symbol, 'limit': limit, 'page': page}
-
-            logger.info(f"Requesting FMP data for {symbol}, page {page}...")
-            data = await self._make_request(session, 'fmp', endpoint, params)
-
-            if not data:
-                break  # Stop if API returns no more data
-
-            for trade in data:
-                # IMPROVEMENT: Check the transaction date. If it's too old, stop fetching.
-                trade_date_str = trade.get('transactionDate')
-                try:
-                    trade_date = datetime.strptime(trade_date_str, '%Y-%m-%d %H:%M:%S')
-                    if trade_date < start_date:
-                        keep_fetching = False
-                        # Don't process this trade, and signal to stop fetching more pages
-                        break
-                except (ValueError, TypeError):
-                    # If date is invalid, we can't check it, so we process it but log a warning
-                    logger.warning(f"Could not parse date for FMP trade: {trade_date_str}")
-
-                try:
-                    disposition = (trade.get('acquistionOrDisposition') or '').upper()
-                    trans_type = TransactionType.BUY.value if disposition == 'A' else TransactionType.SELL.value
-
-                    shares = int(trade.get('securitiesTransacted', 0) or 0)
-                    price = float(trade.get('price', 0) or 0)
-
-                    insider_trade = InsiderTrade(
-                        symbol=trade.get('symbol', ''),
-                        company_name=trade.get('companyName', ''),
-                        insider_name=trade.get('reportingName', ''),
-                        position=trade.get('typeOfOwner', ''),
-                        transaction_date=trade.get('transactionDate', ''),
-                        transaction_type=trans_type,
-                        shares=shares,
-                        price=price,
-                        value=float(shares * price),
-                        form_type=trade.get('formType', ''),
-                        source='FMP',
-                        filing_date=trade.get('filingDate', ''),
-                        ownership_type=trade.get('directOrIndirectOwnership', '')
-                    )
-                    all_trades.append(insider_trade)
-                except (ValueError, TypeError) as e:
-                    logger.warning(f"Error parsing FMP trade data for {symbol}: {e} -> {trade}")
-                    continue
-
-            page += 1
-            await asyncio.sleep(0.2)
-
-        logger.info(f"Retrieved {len(all_trades)} recent trades from FMP for {symbol}.")
+        today = datetime.now()
+        date_range = [today - timedelta(days=i) for i in range(filter_days)]
+
+        # Outer Loop: Iterate through each day in the specified range.
+        for single_date in date_range:
+            page = 0
+            date_str = single_date.strftime('%Y-%m-%d')
+
+            # Inner Loop: Handle pagination for the current day.
+            while True:
+                endpoint = "insider-trading"
+                params = {
+                    'symbol': symbol,
+                    'date': date_str,
+                    'page': page,
+                    'limit': limit
+                }
+
+                logger.info(f"Requesting FMP data for {symbol} on {date_str}, page {page}...")
+                data = await self._make_request(session, 'fmp', endpoint, params)
+
+                # If no data is returned for this page, we're done with this date.
+                if not data:
+                    break
+
+                for trade in data:
+                    try:
+                        disposition = (trade.get('acquistionOrDisposition') or '').upper()
+                        trans_type = TransactionType.BUY.value if disposition == 'A' else TransactionType.SELL.value
+
+                        shares = int(trade.get('securitiesTransacted', 0) or 0)
+                        price = float(trade.get('price', 0) or 0)
+
+                        insider_trade = InsiderTrade(
+                            symbol=trade.get('symbol', ''),
+                            company_name=trade.get('companyName', ''),
+                            insider_name=trade.get('reportingName', ''),
+                            position=trade.get('typeOfOwner', ''),
+                            transaction_date=trade.get('transactionDate', ''),
+                            transaction_type=trans_type,
+                            shares=shares,
+                            price=price,
+                            value=float(shares * price),
+                            form_type=trade.get('formType', ''),
+                            source='FMP',
+                            filing_date=trade.get('filingDate', ''),
+                            ownership_type=trade.get('directOrIndirectOwnership', '')
+                        )
+                        all_trades.append(insider_trade)
+                    except (ValueError, TypeError) as e:
+                        logger.warning(f"Error parsing FMP trade data for {symbol}: {e} -> {trade}")
+                        continue
+
+                # Increment the page to fetch the next set of results for the same day.
+                page += 1
+                await asyncio.sleep(0.2)  # Courteous delay
+
+        logger.info(
+            f"Retrieved {len(all_trades)} trades from FMP for {symbol} by iterating through {len(date_range)} days.")
         return all_trades
 
     async def get_sec_api_insider_trades(self, session: aiohttp.ClientSession,