This commit is contained in:
@@ -0,0 +1,12 @@
|
||||
node_modules
|
||||
npm-debug.log
|
||||
.git
|
||||
.gitignore
|
||||
.env
|
||||
.DS_Store
|
||||
*.md
|
||||
!README.md
|
||||
.vscode
|
||||
.idea
|
||||
coverage
|
||||
.eslintcache
|
||||
+2
-2
@@ -1,2 +1,2 @@
|
||||
GHOSTFOLIO_SECURITY_TOKEN=
|
||||
GHOSTFOLIO_HOST=
|
||||
# Optional: Port to run the server on (defaults to 3000)
|
||||
PORT=3000
|
||||
|
||||
@@ -2,30 +2,48 @@ name: Build and publish
|
||||
run-name: Build docker image and publish to registry
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 10,22 * * *'
|
||||
push:
|
||||
branches:
|
||||
- 'main'
|
||||
tags:
|
||||
- 'v*'
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ghcr.io/catthehacker/ubuntu:act-latest
|
||||
steps:
|
||||
- run: apt update
|
||||
- run: apt install -y libatk1.0-0 libatk-bridge2.0-0 libasound2t64 libgdk-pixbuf2.0-0 libgtk-3-0 libgbm-dev libnss3-dev libxss-dev
|
||||
name: Prepare environment
|
||||
- uses: https://github.com/actions/checkout@v4
|
||||
- name: Use Node.js
|
||||
uses: https://github.com/actions/setup-node@v3
|
||||
- name: Checkout code
|
||||
uses: https://github.com/actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: https://github.com/docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Gitea Container Registry
|
||||
uses: https://github.com/docker/login-action@v3
|
||||
with:
|
||||
node-version: '20.18'
|
||||
- run: npm i
|
||||
name: Install dependencies
|
||||
- run: node index.mjs
|
||||
name: Run app
|
||||
env:
|
||||
GHOSTFOLIO_SECURITY_TOKEN: ${{ secrets.GHOSTFOLIO_SECURITY_TOKEN }}
|
||||
GHOSTFOLIO_HOST: ${{ secrets.GHOSTFOLIO_HOST }}
|
||||
registry: 10.0.2.28:222
|
||||
username: lino
|
||||
password: ${{ secrets.REGISTRY_PASSWORD }}
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: https://github.com/docker/metadata-action@v5
|
||||
with:
|
||||
images: 10.0.2.28:222/${{ gitea.repository }}
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
type=ref,event=pr
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=sha,prefix={{branch}}-
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: https://github.com/docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
# Use Node.js LTS with Debian (needed for Puppeteer dependencies)
|
||||
FROM node:20-bookworm-slim
|
||||
|
||||
# Install Chromium dependencies for Puppeteer
|
||||
RUN apt-get update && apt-get install -y \
|
||||
chromium \
|
||||
fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-freefont-ttf \
|
||||
libxss1 \
|
||||
--no-install-recommends \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Set Puppeteer to use installed Chromium
|
||||
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
|
||||
ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
|
||||
|
||||
# Create app directory
|
||||
WORKDIR /usr/src/app
|
||||
|
||||
# Copy package files
|
||||
COPY package*.json ./
|
||||
|
||||
# Install dependencies
|
||||
RUN npm ci --only=production
|
||||
|
||||
# Copy application files
|
||||
COPY . .
|
||||
|
||||
# Expose port
|
||||
EXPOSE 3000
|
||||
|
||||
# Health check
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \
|
||||
CMD node -e "require('http').get('http://localhost:3000/health', (r) => {process.exit(r.statusCode === 200 ? 0 : 1)})"
|
||||
|
||||
# Run the application
|
||||
CMD ["npm", "start"]
|
||||
@@ -1,21 +1,97 @@
|
||||
# bpi-stock-price-scraper
|
||||
|
||||
Extrair preço de PPRs para introduzir no Ghostfolio
|
||||
Web server API to fetch latest BPI PPR stock prices on demand.
|
||||
|
||||
# Utilização
|
||||
## Features
|
||||
|
||||
Criar ficheiro `.env` a partir do `.env.example`
|
||||
- REST API endpoints to get latest prices for BPI Destino PPR funds
|
||||
- Returns price data as JSON
|
||||
- Dockerized for easy deployment
|
||||
- Health check endpoint included
|
||||
|
||||
## Docker
|
||||
## Available Symbols
|
||||
|
||||
```
|
||||
docker compose build
|
||||
docker compose up
|
||||
- `BPIDEST2040` - BPI Destino PPR 2040
|
||||
- `BPIDEST2050` - BPI Destino PPR 2050
|
||||
|
||||
## API Endpoints
|
||||
|
||||
- `GET /` - API information and usage examples
|
||||
- `GET /health` - Health check with available symbols
|
||||
- `GET /price/:symbol` - Get latest price for a specific symbol
|
||||
|
||||
### Example Response
|
||||
|
||||
```json
|
||||
{
|
||||
"symbol": "BPIDEST2040",
|
||||
"name": "BPI Destino PPR 2040",
|
||||
"price": 7.73313,
|
||||
"date": "2026-04-09",
|
||||
"timestamp": "2026-04-10T16:11:25.850Z"
|
||||
}
|
||||
```
|
||||
|
||||
## Node
|
||||
## Usage
|
||||
|
||||
### Docker (Recommended)
|
||||
|
||||
Build and run with docker-compose:
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
yarn
|
||||
node index.mjs
|
||||
|
||||
Or build and run manually:
|
||||
|
||||
```bash
|
||||
docker build -t bpi-stock-price-scraper .
|
||||
docker run -p 3000:3000 bpi-stock-price-scraper
|
||||
```
|
||||
|
||||
### Node.js
|
||||
|
||||
Install dependencies and run:
|
||||
|
||||
```bash
|
||||
npm install
|
||||
npm start
|
||||
```
|
||||
|
||||
For development with auto-reload:
|
||||
|
||||
```bash
|
||||
npm run dev
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
The server runs on port 3000 by default. You can change it via the `PORT` environment variable:
|
||||
|
||||
```bash
|
||||
PORT=8080 npm start
|
||||
```
|
||||
|
||||
Or in docker-compose.yml:
|
||||
|
||||
```yaml
|
||||
environment:
|
||||
- PORT=8080
|
||||
ports:
|
||||
- "8080:8080"
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
Test the API:
|
||||
|
||||
```bash
|
||||
# Get API info
|
||||
curl http://localhost:3000/
|
||||
|
||||
# Check health
|
||||
curl http://localhost:3000/health
|
||||
|
||||
# Get latest price
|
||||
curl http://localhost:3000/price/BPIDEST2040
|
||||
```
|
||||
@@ -0,0 +1,16 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
bpi-scraper:
|
||||
build: .
|
||||
ports:
|
||||
- "3000:3000"
|
||||
environment:
|
||||
- PORT=3000
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "node", "-e", "require('http').get('http://localhost:3000/health', (r) => {process.exit(r.statusCode === 200 ? 0 : 1)})"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 40s
|
||||
@@ -1,33 +1,44 @@
|
||||
import puppeteer from "puppeteer";
|
||||
import process from "node:process";
|
||||
import axios from "axios";
|
||||
import express from "express";
|
||||
import "dotenv/config";
|
||||
|
||||
const contas = {
|
||||
bpi2040: {
|
||||
const app = express();
|
||||
const PORT = process.env.PORT || 3000;
|
||||
|
||||
// Symbol configuration
|
||||
const symbols = {
|
||||
BPIDEST2040: {
|
||||
url: "https://www.bancobpi.pt/particulares/poupar-investir/ppr/bpi-destino-ppr-2040",
|
||||
ghostfolioName: "GF_BPI_Destino_PPR_2040",
|
||||
name: "BPI Destino PPR 2040",
|
||||
},
|
||||
bpi2050: {
|
||||
BPIDEST2050: {
|
||||
url: "https://www.bancobpi.pt/particulares/poupar-investir/ppr/bpi-destino-ppr-2050",
|
||||
ghostfolioName: "GF_BPI_Destino_PPR_2050",
|
||||
name: "BPI Destino PPR 2050",
|
||||
},
|
||||
};
|
||||
|
||||
async function parseLogRocketBlogHome(conta) {
|
||||
// Function to scrape price for a given symbol
|
||||
async function getLatestPrice(symbol) {
|
||||
const config = symbols[symbol];
|
||||
|
||||
if (!config) {
|
||||
throw new Error(`Invalid symbol: ${symbol}`);
|
||||
}
|
||||
|
||||
// Launch the browser
|
||||
const browser = await puppeteer.launch({
|
||||
args: ["--no-sandbox"],
|
||||
timeout: 10000,
|
||||
});
|
||||
|
||||
try {
|
||||
// Open a new tab
|
||||
const page = await browser.newPage();
|
||||
|
||||
// Visit the page and wait until network connections are completed
|
||||
await page.goto(conta.url, { waitUntil: "networkidle2" });
|
||||
await page.goto(config.url, { waitUntil: "networkidle2" });
|
||||
|
||||
// Interact with the DOM to retrieve the titles
|
||||
// Interact with the DOM to retrieve the price and date
|
||||
const [price, date] = await page.evaluate(() => {
|
||||
const spanTags = document.getElementsByTagName("span");
|
||||
const priceSearchText = "ÚLTIMA COTAÇÃO:";
|
||||
@@ -38,66 +49,80 @@ async function parseLogRocketBlogHome(conta) {
|
||||
for (let i = 0; i < spanTags.length; i++) {
|
||||
if (spanTags[i].textContent.trim() == priceSearchText) {
|
||||
priceElementFound = spanTags[i];
|
||||
console.log(`Found ${priceSearchText}.`);
|
||||
if (priceElementFound && dateElementFound) break;
|
||||
}
|
||||
if (spanTags[i].textContent.trim() == dateSearchText) {
|
||||
dateElementFound = spanTags[i];
|
||||
console.log(`Found ${dateSearchText}.`);
|
||||
if (priceElementFound && dateElementFound) break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!priceElementFound || !dateElementFound) {
|
||||
throw new Error("Could not find price or date elements");
|
||||
}
|
||||
|
||||
return [
|
||||
priceElementFound.nextSibling.innerHTML,
|
||||
dateElementFound.nextSibling.innerHTML,
|
||||
];
|
||||
});
|
||||
|
||||
// Don't forget to close the browser instance to clean up the memory
|
||||
await browser.close();
|
||||
|
||||
const marketPrice = parseFloat(
|
||||
price.replace("€", "").trim().replace(",", "."),
|
||||
);
|
||||
|
||||
// Print the results
|
||||
console.log(decodeURIComponent(conta.ghostfolioName));
|
||||
console.log(`Current price: € ${marketPrice}`);
|
||||
console.log(`Date: ${date}`);
|
||||
|
||||
const bearerTokenResponse = await axios.post(
|
||||
`${process.env.GHOSTFOLIO_HOST}/api/v1/auth/anonymous`,
|
||||
{
|
||||
accessToken: process.env.GHOSTFOLIO_SECURITY_TOKEN,
|
||||
},
|
||||
);
|
||||
|
||||
const response = await axios
|
||||
.post(
|
||||
`${process.env.GHOSTFOLIO_HOST}/api/v1/market-data/MANUAL/${conta.ghostfolioName}`,
|
||||
{
|
||||
marketData: [
|
||||
{
|
||||
date,
|
||||
marketPrice,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${bearerTokenResponse.data.authToken}`,
|
||||
},
|
||||
},
|
||||
)
|
||||
.catch(console.error);
|
||||
|
||||
console.log(response.status);
|
||||
return {
|
||||
symbol,
|
||||
name: config.name,
|
||||
price: marketPrice,
|
||||
date: date.trim(),
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
} finally {
|
||||
// Always close the browser
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
await Promise.all([
|
||||
parseLogRocketBlogHome(contas.bpi2040),
|
||||
parseLogRocketBlogHome(contas.bpi2050),
|
||||
]);
|
||||
// API endpoint to get latest price for a symbol
|
||||
app.get("/price/:symbol", async (req, res) => {
|
||||
const { symbol } = req.params;
|
||||
|
||||
process.exit(0);
|
||||
try {
|
||||
console.log(`Fetching price for symbol: ${symbol}`);
|
||||
const priceData = await getLatestPrice(symbol);
|
||||
console.log(`Success: ${symbol} - €${priceData.price}`);
|
||||
res.json(priceData);
|
||||
} catch (error) {
|
||||
console.error(`Error fetching price for ${symbol}:`, error.message);
|
||||
res.status(400).json({
|
||||
error: error.message,
|
||||
symbol,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Health check endpoint
|
||||
app.get("/health", (req, res) => {
|
||||
res.json({ status: "ok", availableSymbols: Object.keys(symbols) });
|
||||
});
|
||||
|
||||
// Root endpoint with usage instructions
|
||||
app.get("/", (req, res) => {
|
||||
res.json({
|
||||
message: "BPI Stock Price Scraper API",
|
||||
endpoints: {
|
||||
health: "/health",
|
||||
price: "/price/:symbol",
|
||||
},
|
||||
availableSymbols: Object.keys(symbols),
|
||||
example: `/price/BPIDEST2040`,
|
||||
});
|
||||
});
|
||||
|
||||
// Start the server
|
||||
app.listen(PORT, () => {
|
||||
console.log(`Server running on http://localhost:${PORT}`);
|
||||
console.log(`Available symbols: ${Object.keys(symbols).join(", ")}`);
|
||||
console.log(`Example: http://localhost:${PORT}/price/BPIDEST2040`);
|
||||
});
|
||||
|
||||
Generated
+852
-1
File diff suppressed because it is too large
Load Diff
+7
-2
@@ -1,13 +1,18 @@
|
||||
{
|
||||
"name": "bpi-stock-price-scraper",
|
||||
"version": "1.0.0",
|
||||
"main": "index.js",
|
||||
"main": "index.mjs",
|
||||
"type": "module",
|
||||
"repository": "ssh://git@10.0.2.28:222/lino-authelia/bpi-stock-price-scraper.git",
|
||||
"author": "Lino Silva <me@lino.cooking>",
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
"start": "node index.mjs",
|
||||
"dev": "node --watch index.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"axios": "^1.7.7",
|
||||
"dotenv": "^16.4.5",
|
||||
"express": "^5.2.1",
|
||||
"puppeteer": "^23.5.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
||||
Generated
+2002
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user