import puppeteer, { Page } from "puppeteer"; import express from "express"; import "dotenv/config"; import { Provider, ProviderType } from "./interfaces"; import { availableSymbols } from "./availableSymbols"; const app = express(); const PORT = process.env.PORT || 3000; const providers: Record = { [ProviderType.BPI]: { type: ProviderType.BPI, baseUrl: "https://www.bancobpi.pt/particulares/poupar-investir/ppr/${symbol}", }, [ProviderType.DECO]: { type: ProviderType.DECO, baseUrl: "https://www.deco.proteste.pt/investe/investimentos/fundos/${symbol}", }, [ProviderType.FT]: { type: ProviderType.FT, baseUrl: "https://markets.ft.com/data/funds/tearsheet/summary?s=${symbol}", }, }; interface SymbolConfig { name: string; type: ProviderType; url: string; } function getConfigForSymbol(symbol: string): SymbolConfig { if (!availableSymbols[symbol]) { throw new Error(`Symbol ${symbol} is not supported`); } return { name: symbol, type: availableSymbols[symbol].provider, url: providers[availableSymbols[symbol].provider].baseUrl.replace( "${symbol}", availableSymbols[symbol].mappedSymbol, ), }; } // Function to scrape price for a given symbol async function getLatestPrice(symbol: string) { const config: SymbolConfig = getConfigForSymbol(symbol); console.log( `Scraping ${config.url} for symbol ${symbol} using provider ${config.type}`, ); // Launch the browser const browser = await puppeteer.launch({ args: ["--no-sandbox"], timeout: 10000, }); try { // Open a new tab const page = await browser.newPage(); // Visit the page and wait until network connections are completed await page.goto(config.url, { waitUntil: "networkidle2" }); switch (config.type) { case ProviderType.BPI: return await scrapeBPI(page, symbol, config); case ProviderType.DECO: return await scrapeDeco(page, symbol, config); case ProviderType.FT: return await scrapeFT(page, symbol, config); default: throw new Error(`Unsupported type: ${config.type}`); } } finally { // Always close the browser await browser.close(); } } async function scrapeDeco(page: Page, symbol: string, config: SymbolConfig) { // Interact with the DOM to retrieve the price and date const [price, date] = await page.evaluate(() => { const priceElement = document.querySelector( ".product-points-data__current-value", ); const dateElement = document.querySelector( ".product-points-data__current-value + span", ); if (!priceElement || !dateElement) { throw new Error("Could not find price or date elements"); } const price = priceElement.textContent .replace("€", "") .replace("EUR", "") .replace(",", ".") .trim(); const date = dateElement.textContent.trim(); return [price, date]; }); const marketPrice = parseFloat(price); return { symbol, name: config.name, currency: "EUR", price: marketPrice, date, timestamp: new Date().toISOString(), }; } async function scrapeBPI(page: Page, symbol: string, config: SymbolConfig) { // Interact with the DOM to retrieve the price and date const [price, date] = await page.evaluate(() => { const spanTags = document.getElementsByTagName("span"); const priceSearchText = "ÚLTIMA COTAÇÃO:"; const dateSearchText = "DATA COTAÇÃO:"; let priceElementFound; let dateElementFound; for (let i = 0; i < spanTags.length; i++) { if (spanTags[i].textContent.trim() == priceSearchText) { priceElementFound = spanTags[i]; if (priceElementFound && dateElementFound) break; } if (spanTags[i].textContent.trim() == dateSearchText) { dateElementFound = spanTags[i]; if (priceElementFound && dateElementFound) break; } } if (!priceElementFound || !dateElementFound) { throw new Error("Could not find price or date elements"); } return [ priceElementFound?.nextSibling?.textContent, dateElementFound?.nextSibling?.textContent, ]; }); const marketPrice = parseFloat( price?.replace("€", "").trim().replace(",", ".") || "0", ); return { symbol, name: config.name, currency: "EUR", price: marketPrice, date: date?.trim() || "", timestamp: new Date().toISOString(), }; } async function scrapeFT(page: Page, symbol: string, config: SymbolConfig) { // Interact with the DOM to retrieve the price and date const price = await page.evaluate(() => { const spanTags = document.getElementsByTagName("span"); const priceSearchText = "Price (EUR)"; let priceElementFound; for (let i = 0; i < spanTags.length; i++) { if (spanTags[i].textContent.trim() == priceSearchText) { priceElementFound = spanTags[i]; if (priceElementFound) break; } } if (!priceElementFound) { throw new Error("Could not find price element"); } return priceElementFound?.nextSibling?.textContent; }); const marketPrice = parseFloat( price?.replace("€", "").trim().replace(",", ".") || "0", ); return { symbol, name: config.name, currency: "EUR", price: marketPrice, // date in FT is not available, so we use the current date as a placeholder // yyyy-mm-dd format date: new Date().toISOString().split("T")[0], timestamp: new Date().toISOString(), }; } // API endpoint to get latest price for a symbol app.get("/price/:symbol", async (req, res) => { const { symbol } = req.params; try { console.log(`Fetching price for symbol: ${symbol}`); const priceData = await getLatestPrice(symbol); console.log(`Success: ${symbol} - ${priceData.price}`); res.json(priceData); } catch (error: any) { console.error(`Error fetching price for ${symbol}:`, error.message); res.status(400).json({ error: error.message, symbol, }); } }); // Health check endpoint app.get("/health", (req, res) => { res.json({ status: "ok" }); }); // Root endpoint with usage instructions app.get("/", (req, res) => { res.json({ message: "BPI Stock Price Scraper API", endpoints: { health: "/health", "price-bpi": "/price/bpi/:symbol", "price-deco": "/price/deco/:symbol", "price-ft": "/price/ft/:symbol", }, example: `/price/bpi/BPIDEST2040`, }); }); // Start the server app.listen(PORT, () => { console.log(`Server running on http://localhost:${PORT}`); console.log("Available symbols:", Object.keys(availableSymbols).join(", ")); console.log(`Example: http://localhost:${PORT}/price/bpi/BPIDEST2040`); });