bpi-stock-price-scraper/index.ts

import puppeteer, { Page } from "puppeteer";
import express from "express";
import "dotenv/config";
import { Provider, ProviderType } from "./interfaces";
import { availableSymbols } from "./availableSymbols";

const app = express();
const PORT = process.env.PORT || 3000;

const providers: Record<ProviderType, Provider> = {
  [ProviderType.BPI]: {
    type: ProviderType.BPI,
    baseUrl:
      "https://www.bancobpi.pt/particulares/poupar-investir/ppr/${symbol}",
  },
  [ProviderType.DECO]: {
    type: ProviderType.DECO,
    baseUrl:
      "https://www.deco.proteste.pt/investe/investimentos/fundos/${symbol}",
  },
  [ProviderType.FT]: {
    type: ProviderType.FT,
    baseUrl: "https://markets.ft.com/data/funds/tearsheet/summary?s=${symbol}",
  },
};

interface SymbolConfig {
  name: string;
  type: ProviderType;
  url: string;
}

function getConfigForSymbol(symbol: string): SymbolConfig {
  if (!availableSymbols[symbol]) {
    throw new Error(`Symbol ${symbol} is not supported`);
  }

  return {
    name: symbol,
    type: availableSymbols[symbol].provider,
    url: providers[availableSymbols[symbol].provider].baseUrl.replace(
      "${symbol}",
      availableSymbols[symbol].mappedSymbol,
    ),
  };
}

// Function to scrape price for a given symbol
async function getLatestPrice(symbol: string) {
  const config: SymbolConfig = getConfigForSymbol(symbol);

  console.log(
    `Scraping ${config.url} for symbol ${symbol} using provider ${config.type}`,
  );

  // Launch the browser
  const browser = await puppeteer.launch({
    args: ["--no-sandbox"],
    timeout: 10000,
  });

  try {
    // Open a new tab
    const page = await browser.newPage();

    // Visit the page and wait until network connections are completed
    await page.goto(config.url, { waitUntil: "networkidle2" });

    switch (config.type) {
      case ProviderType.BPI:
        return await scrapeBPI(page, symbol, config);
      case ProviderType.DECO:
        return await scrapeDeco(page, symbol, config);
      case ProviderType.FT:
        return await scrapeFT(page, symbol, config);
      default:
        throw new Error(`Unsupported type: ${config.type}`);
    }
  } finally {
    // Always close the browser
    await browser.close();
  }
}

async function scrapeDeco(page: Page, symbol: string, config: SymbolConfig) {
  // Interact with the DOM to retrieve the price and date
  const [price, date] = await page.evaluate(() => {
    const priceElement = document.querySelector(
      ".product-points-data__current-value",
    );
    const dateElement = document.querySelector(
      ".product-points-data__current-value + span",
    );

    if (!priceElement || !dateElement) {
      throw new Error("Could not find price or date elements");
    }

    const price = priceElement.textContent
      .replace("€", "")
      .replace("EUR", "")
      .replace(",", ".")
      .trim();
    const date = dateElement.textContent.trim();

    return [price, date];
  });

  const marketPrice = parseFloat(price);
  return {
    symbol,
    name: config.name,
    currency: "EUR",
    price: marketPrice,
    date,
    timestamp: new Date().toISOString(),
  };
}

async function scrapeBPI(page: Page, symbol: string, config: SymbolConfig) {
  // Interact with the DOM to retrieve the price and date
  const [price, date] = await page.evaluate(() => {
    const spanTags = document.getElementsByTagName("span");
    const priceSearchText = "ÚLTIMA COTAÇÃO:";
    const dateSearchText = "DATA COTAÇÃO:";
    let priceElementFound;
    let dateElementFound;

    for (let i = 0; i < spanTags.length; i++) {
      if (spanTags[i].textContent.trim() == priceSearchText) {
        priceElementFound = spanTags[i];
        if (priceElementFound && dateElementFound) break;
      }
      if (spanTags[i].textContent.trim() == dateSearchText) {
        dateElementFound = spanTags[i];
        if (priceElementFound && dateElementFound) break;
      }
    }

    if (!priceElementFound || !dateElementFound) {
      throw new Error("Could not find price or date elements");
    }

    return [
      priceElementFound?.nextSibling?.textContent,
      dateElementFound?.nextSibling?.textContent,
    ];
  });

  const marketPrice = parseFloat(
    price?.replace("€", "").trim().replace(",", ".") || "0",
  );

  return {
    symbol,
    name: config.name,
    currency: "EUR",
    price: marketPrice,
    date: date?.trim() || "",
    timestamp: new Date().toISOString(),
  };
}

async function scrapeFT(page: Page, symbol: string, config: SymbolConfig) {
  // Interact with the DOM to retrieve the price and date
  const price = await page.evaluate(() => {
    const spanTags = document.getElementsByTagName("span");
    const priceSearchText = "Price (EUR)";
    let priceElementFound;

    for (let i = 0; i < spanTags.length; i++) {
      if (spanTags[i].textContent.trim() == priceSearchText) {
        priceElementFound = spanTags[i];
        if (priceElementFound) break;
      }
    }

    if (!priceElementFound) {
      throw new Error("Could not find price element");
    }

    return priceElementFound?.nextSibling?.textContent;
  });

  const marketPrice = parseFloat(
    price?.replace("€", "").trim().replace(",", ".") || "0",
  );

  return {
    symbol,
    name: config.name,
    currency: "EUR",
    price: marketPrice,
    // date in FT is not available, so we use the current date as a placeholder
    // yyyy-mm-dd format
    date: new Date().toISOString().split("T")[0],
    timestamp: new Date().toISOString(),
  };
}

// API endpoint to get latest price for a symbol
app.get("/price/:symbol", async (req, res) => {
  const { symbol } = req.params;

  try {
    console.log(`Fetching price for symbol: ${symbol}`);
    const priceData = await getLatestPrice(symbol);
    console.log(`Success: ${symbol} - ${priceData.price}`);
    res.json(priceData);
  } catch (error: any) {
    console.error(`Error fetching price for ${symbol}:`, error.message);
    res.status(400).json({
      error: error.message,
      symbol,
    });
  }
});

// Health check endpoint
app.get("/health", (req, res) => {
  res.json({ status: "ok" });
});

// Root endpoint with usage instructions
app.get("/", (req, res) => {
  res.json({
    message: "BPI Stock Price Scraper API",
    endpoints: {
      health: "/health",
      "price-bpi": "/price/bpi/:symbol",
      "price-deco": "/price/deco/:symbol",
      "price-ft": "/price/ft/:symbol",
    },
    example: `/price/bpi/BPIDEST2040`,
  });
});

// Start the server
app.listen(PORT, () => {
  console.log(`Server running on http://localhost:${PORT}`);
  console.log("Available symbols:", Object.keys(availableSymbols).join(", "));
  console.log(`Example: http://localhost:${PORT}/price/bpi/BPIDEST2040`);
});