This commit is contained in:
@@ -0,0 +1,12 @@
|
|||||||
|
node_modules
|
||||||
|
npm-debug.log
|
||||||
|
.git
|
||||||
|
.gitignore
|
||||||
|
.env
|
||||||
|
.DS_Store
|
||||||
|
*.md
|
||||||
|
!README.md
|
||||||
|
.vscode
|
||||||
|
.idea
|
||||||
|
coverage
|
||||||
|
.eslintcache
|
||||||
+2
-2
@@ -1,2 +1,2 @@
|
|||||||
GHOSTFOLIO_SECURITY_TOKEN=
|
# Optional: Port to run the server on (defaults to 3000)
|
||||||
GHOSTFOLIO_HOST=
|
PORT=3000
|
||||||
|
|||||||
@@ -2,30 +2,48 @@ name: Build and publish
|
|||||||
run-name: Build docker image and publish to registry
|
run-name: Build docker image and publish to registry
|
||||||
|
|
||||||
on:
|
on:
|
||||||
schedule:
|
|
||||||
- cron: '0 10,22 * * *'
|
|
||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- 'main'
|
- 'main'
|
||||||
|
tags:
|
||||||
|
- 'v*'
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build:
|
build:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
container:
|
|
||||||
image: ghcr.io/catthehacker/ubuntu:act-latest
|
|
||||||
steps:
|
steps:
|
||||||
- run: apt update
|
- name: Checkout code
|
||||||
- run: apt install -y libatk1.0-0 libatk-bridge2.0-0 libasound2t64 libgdk-pixbuf2.0-0 libgtk-3-0 libgbm-dev libnss3-dev libxss-dev
|
uses: https://github.com/actions/checkout@v4
|
||||||
name: Prepare environment
|
|
||||||
- uses: https://github.com/actions/checkout@v4
|
- name: Set up Docker Buildx
|
||||||
- name: Use Node.js
|
uses: https://github.com/docker/setup-buildx-action@v3
|
||||||
uses: https://github.com/actions/setup-node@v3
|
|
||||||
|
- name: Log in to Gitea Container Registry
|
||||||
|
uses: https://github.com/docker/login-action@v3
|
||||||
with:
|
with:
|
||||||
node-version: '20.18'
|
registry: 10.0.2.28:222
|
||||||
- run: npm i
|
username: lino
|
||||||
name: Install dependencies
|
password: ${{ secrets.REGISTRY_PASSWORD }}
|
||||||
- run: node index.mjs
|
|
||||||
name: Run app
|
- name: Extract metadata
|
||||||
env:
|
id: meta
|
||||||
GHOSTFOLIO_SECURITY_TOKEN: ${{ secrets.GHOSTFOLIO_SECURITY_TOKEN }}
|
uses: https://github.com/docker/metadata-action@v5
|
||||||
GHOSTFOLIO_HOST: ${{ secrets.GHOSTFOLIO_HOST }}
|
with:
|
||||||
|
images: 10.0.2.28:222/${{ gitea.repository }}
|
||||||
|
tags: |
|
||||||
|
type=ref,event=branch
|
||||||
|
type=ref,event=pr
|
||||||
|
type=semver,pattern={{version}}
|
||||||
|
type=semver,pattern={{major}}.{{minor}}
|
||||||
|
type=sha,prefix={{branch}}-
|
||||||
|
type=raw,value=latest,enable={{is_default_branch}}
|
||||||
|
|
||||||
|
- name: Build and push Docker image
|
||||||
|
uses: https://github.com/docker/build-push-action@v5
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
push: true
|
||||||
|
tags: ${{ steps.meta.outputs.tags }}
|
||||||
|
labels: ${{ steps.meta.outputs.labels }}
|
||||||
|
cache-from: type=gha
|
||||||
|
cache-to: type=gha,mode=max
|
||||||
|
|||||||
+36
@@ -0,0 +1,36 @@
|
|||||||
|
# Use Node.js LTS with Debian (needed for Puppeteer dependencies)
|
||||||
|
FROM node:20-bookworm-slim
|
||||||
|
|
||||||
|
# Install Chromium dependencies for Puppeteer
|
||||||
|
RUN apt-get update && apt-get install -y \
|
||||||
|
chromium \
|
||||||
|
fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-freefont-ttf \
|
||||||
|
libxss1 \
|
||||||
|
--no-install-recommends \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Set Puppeteer to use installed Chromium
|
||||||
|
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
|
||||||
|
ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
|
||||||
|
|
||||||
|
# Create app directory
|
||||||
|
WORKDIR /usr/src/app
|
||||||
|
|
||||||
|
# Copy package files
|
||||||
|
COPY package*.json ./
|
||||||
|
|
||||||
|
# Install dependencies
|
||||||
|
RUN npm ci --only=production
|
||||||
|
|
||||||
|
# Copy application files
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
# Expose port
|
||||||
|
EXPOSE 3000
|
||||||
|
|
||||||
|
# Health check
|
||||||
|
HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \
|
||||||
|
CMD node -e "require('http').get('http://localhost:3000/health', (r) => {process.exit(r.statusCode === 200 ? 0 : 1)})"
|
||||||
|
|
||||||
|
# Run the application
|
||||||
|
CMD ["npm", "start"]
|
||||||
@@ -1,21 +1,97 @@
|
|||||||
# bpi-stock-price-scraper
|
# bpi-stock-price-scraper
|
||||||
|
|
||||||
Extrair preço de PPRs para introduzir no Ghostfolio
|
Web server API to fetch latest BPI PPR stock prices on demand.
|
||||||
|
|
||||||
# Utilização
|
## Features
|
||||||
|
|
||||||
Criar ficheiro `.env` a partir do `.env.example`
|
- REST API endpoints to get latest prices for BPI Destino PPR funds
|
||||||
|
- Returns price data as JSON
|
||||||
|
- Dockerized for easy deployment
|
||||||
|
- Health check endpoint included
|
||||||
|
|
||||||
## Docker
|
## Available Symbols
|
||||||
|
|
||||||
```
|
- `BPIDEST2040` - BPI Destino PPR 2040
|
||||||
docker compose build
|
- `BPIDEST2050` - BPI Destino PPR 2050
|
||||||
docker compose up
|
|
||||||
|
## API Endpoints
|
||||||
|
|
||||||
|
- `GET /` - API information and usage examples
|
||||||
|
- `GET /health` - Health check with available symbols
|
||||||
|
- `GET /price/:symbol` - Get latest price for a specific symbol
|
||||||
|
|
||||||
|
### Example Response
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"symbol": "BPIDEST2040",
|
||||||
|
"name": "BPI Destino PPR 2040",
|
||||||
|
"price": 7.73313,
|
||||||
|
"date": "2026-04-09",
|
||||||
|
"timestamp": "2026-04-10T16:11:25.850Z"
|
||||||
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
## Node
|
## Usage
|
||||||
|
|
||||||
|
### Docker (Recommended)
|
||||||
|
|
||||||
|
Build and run with docker-compose:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
Or build and run manually:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker build -t bpi-stock-price-scraper .
|
||||||
|
docker run -p 3000:3000 bpi-stock-price-scraper
|
||||||
|
```
|
||||||
|
|
||||||
|
### Node.js
|
||||||
|
|
||||||
|
Install dependencies and run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm install
|
||||||
|
npm start
|
||||||
|
```
|
||||||
|
|
||||||
|
For development with auto-reload:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run dev
|
||||||
|
```
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
The server runs on port 3000 by default. You can change it via the `PORT` environment variable:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
PORT=8080 npm start
|
||||||
|
```
|
||||||
|
|
||||||
|
Or in docker-compose.yml:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
environment:
|
||||||
|
- PORT=8080
|
||||||
|
ports:
|
||||||
|
- "8080:8080"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Testing
|
||||||
|
|
||||||
|
Test the API:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Get API info
|
||||||
|
curl http://localhost:3000/
|
||||||
|
|
||||||
|
# Check health
|
||||||
|
curl http://localhost:3000/health
|
||||||
|
|
||||||
|
# Get latest price
|
||||||
|
curl http://localhost:3000/price/BPIDEST2040
|
||||||
```
|
```
|
||||||
yarn
|
|
||||||
node index.mjs
|
|
||||||
```
|
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
version: '3.8'
|
||||||
|
|
||||||
|
services:
|
||||||
|
bpi-scraper:
|
||||||
|
build: .
|
||||||
|
ports:
|
||||||
|
- "3000:3000"
|
||||||
|
environment:
|
||||||
|
- PORT=3000
|
||||||
|
restart: unless-stopped
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "node", "-e", "require('http').get('http://localhost:3000/health', (r) => {process.exit(r.statusCode === 200 ? 0 : 1)})"]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 10s
|
||||||
|
retries: 3
|
||||||
|
start_period: 40s
|
||||||
@@ -1,103 +1,128 @@
|
|||||||
import puppeteer from "puppeteer";
|
import puppeteer from "puppeteer";
|
||||||
import process from "node:process";
|
import express from "express";
|
||||||
import axios from "axios";
|
|
||||||
import "dotenv/config";
|
import "dotenv/config";
|
||||||
|
|
||||||
const contas = {
|
const app = express();
|
||||||
bpi2040: {
|
const PORT = process.env.PORT || 3000;
|
||||||
|
|
||||||
|
// Symbol configuration
|
||||||
|
const symbols = {
|
||||||
|
BPIDEST2040: {
|
||||||
url: "https://www.bancobpi.pt/particulares/poupar-investir/ppr/bpi-destino-ppr-2040",
|
url: "https://www.bancobpi.pt/particulares/poupar-investir/ppr/bpi-destino-ppr-2040",
|
||||||
ghostfolioName: "GF_BPI_Destino_PPR_2040",
|
name: "BPI Destino PPR 2040",
|
||||||
},
|
},
|
||||||
bpi2050: {
|
BPIDEST2050: {
|
||||||
url: "https://www.bancobpi.pt/particulares/poupar-investir/ppr/bpi-destino-ppr-2050",
|
url: "https://www.bancobpi.pt/particulares/poupar-investir/ppr/bpi-destino-ppr-2050",
|
||||||
ghostfolioName: "GF_BPI_Destino_PPR_2050",
|
name: "BPI Destino PPR 2050",
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
async function parseLogRocketBlogHome(conta) {
|
// Function to scrape price for a given symbol
|
||||||
|
async function getLatestPrice(symbol) {
|
||||||
|
const config = symbols[symbol];
|
||||||
|
|
||||||
|
if (!config) {
|
||||||
|
throw new Error(`Invalid symbol: ${symbol}`);
|
||||||
|
}
|
||||||
|
|
||||||
// Launch the browser
|
// Launch the browser
|
||||||
const browser = await puppeteer.launch({
|
const browser = await puppeteer.launch({
|
||||||
args: ["--no-sandbox"],
|
args: ["--no-sandbox"],
|
||||||
timeout: 10000,
|
timeout: 10000,
|
||||||
});
|
});
|
||||||
|
|
||||||
// Open a new tab
|
try {
|
||||||
const page = await browser.newPage();
|
// Open a new tab
|
||||||
|
const page = await browser.newPage();
|
||||||
|
|
||||||
// Visit the page and wait until network connections are completed
|
// Visit the page and wait until network connections are completed
|
||||||
await page.goto(conta.url, { waitUntil: "networkidle2" });
|
await page.goto(config.url, { waitUntil: "networkidle2" });
|
||||||
|
|
||||||
// Interact with the DOM to retrieve the titles
|
// Interact with the DOM to retrieve the price and date
|
||||||
const [price, date] = await page.evaluate(() => {
|
const [price, date] = await page.evaluate(() => {
|
||||||
const spanTags = document.getElementsByTagName("span");
|
const spanTags = document.getElementsByTagName("span");
|
||||||
const priceSearchText = "ÚLTIMA COTAÇÃO:";
|
const priceSearchText = "ÚLTIMA COTAÇÃO:";
|
||||||
const dateSearchText = "DATA COTAÇÃO:";
|
const dateSearchText = "DATA COTAÇÃO:";
|
||||||
let priceElementFound;
|
let priceElementFound;
|
||||||
let dateElementFound;
|
let dateElementFound;
|
||||||
|
|
||||||
for (let i = 0; i < spanTags.length; i++) {
|
for (let i = 0; i < spanTags.length; i++) {
|
||||||
if (spanTags[i].textContent.trim() == priceSearchText) {
|
if (spanTags[i].textContent.trim() == priceSearchText) {
|
||||||
priceElementFound = spanTags[i];
|
priceElementFound = spanTags[i];
|
||||||
console.log(`Found ${priceSearchText}.`);
|
if (priceElementFound && dateElementFound) break;
|
||||||
if (priceElementFound && dateElementFound) break;
|
}
|
||||||
|
if (spanTags[i].textContent.trim() == dateSearchText) {
|
||||||
|
dateElementFound = spanTags[i];
|
||||||
|
if (priceElementFound && dateElementFound) break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (spanTags[i].textContent.trim() == dateSearchText) {
|
|
||||||
dateElementFound = spanTags[i];
|
if (!priceElementFound || !dateElementFound) {
|
||||||
console.log(`Found ${dateSearchText}.`);
|
throw new Error("Could not find price or date elements");
|
||||||
if (priceElementFound && dateElementFound) break;
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
return [
|
return [
|
||||||
priceElementFound.nextSibling.innerHTML,
|
priceElementFound.nextSibling.innerHTML,
|
||||||
dateElementFound.nextSibling.innerHTML,
|
dateElementFound.nextSibling.innerHTML,
|
||||||
];
|
];
|
||||||
});
|
});
|
||||||
|
|
||||||
// Don't forget to close the browser instance to clean up the memory
|
const marketPrice = parseFloat(
|
||||||
await browser.close();
|
price.replace("€", "").trim().replace(",", "."),
|
||||||
|
);
|
||||||
|
|
||||||
const marketPrice = parseFloat(
|
return {
|
||||||
price.replace("€", "").trim().replace(",", "."),
|
symbol,
|
||||||
);
|
name: config.name,
|
||||||
|
price: marketPrice,
|
||||||
// Print the results
|
date: date.trim(),
|
||||||
console.log(decodeURIComponent(conta.ghostfolioName));
|
timestamp: new Date().toISOString(),
|
||||||
console.log(`Current price: € ${marketPrice}`);
|
};
|
||||||
console.log(`Date: ${date}`);
|
} finally {
|
||||||
|
// Always close the browser
|
||||||
const bearerTokenResponse = await axios.post(
|
await browser.close();
|
||||||
`${process.env.GHOSTFOLIO_HOST}/api/v1/auth/anonymous`,
|
}
|
||||||
{
|
|
||||||
accessToken: process.env.GHOSTFOLIO_SECURITY_TOKEN,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
const response = await axios
|
|
||||||
.post(
|
|
||||||
`${process.env.GHOSTFOLIO_HOST}/api/v1/market-data/MANUAL/${conta.ghostfolioName}`,
|
|
||||||
{
|
|
||||||
marketData: [
|
|
||||||
{
|
|
||||||
date,
|
|
||||||
marketPrice,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
headers: {
|
|
||||||
Authorization: `Bearer ${bearerTokenResponse.data.authToken}`,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.catch(console.error);
|
|
||||||
|
|
||||||
console.log(response.status);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
await Promise.all([
|
// API endpoint to get latest price for a symbol
|
||||||
parseLogRocketBlogHome(contas.bpi2040),
|
app.get("/price/:symbol", async (req, res) => {
|
||||||
parseLogRocketBlogHome(contas.bpi2050),
|
const { symbol } = req.params;
|
||||||
]);
|
|
||||||
|
|
||||||
process.exit(0);
|
try {
|
||||||
|
console.log(`Fetching price for symbol: ${symbol}`);
|
||||||
|
const priceData = await getLatestPrice(symbol);
|
||||||
|
console.log(`Success: ${symbol} - €${priceData.price}`);
|
||||||
|
res.json(priceData);
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`Error fetching price for ${symbol}:`, error.message);
|
||||||
|
res.status(400).json({
|
||||||
|
error: error.message,
|
||||||
|
symbol,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Health check endpoint
|
||||||
|
app.get("/health", (req, res) => {
|
||||||
|
res.json({ status: "ok", availableSymbols: Object.keys(symbols) });
|
||||||
|
});
|
||||||
|
|
||||||
|
// Root endpoint with usage instructions
|
||||||
|
app.get("/", (req, res) => {
|
||||||
|
res.json({
|
||||||
|
message: "BPI Stock Price Scraper API",
|
||||||
|
endpoints: {
|
||||||
|
health: "/health",
|
||||||
|
price: "/price/:symbol",
|
||||||
|
},
|
||||||
|
availableSymbols: Object.keys(symbols),
|
||||||
|
example: `/price/BPIDEST2040`,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// Start the server
|
||||||
|
app.listen(PORT, () => {
|
||||||
|
console.log(`Server running on http://localhost:${PORT}`);
|
||||||
|
console.log(`Available symbols: ${Object.keys(symbols).join(", ")}`);
|
||||||
|
console.log(`Example: http://localhost:${PORT}/price/BPIDEST2040`);
|
||||||
|
});
|
||||||
|
|||||||
Generated
+852
-1
File diff suppressed because it is too large
Load Diff
+7
-2
@@ -1,13 +1,18 @@
|
|||||||
{
|
{
|
||||||
"name": "bpi-stock-price-scraper",
|
"name": "bpi-stock-price-scraper",
|
||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
"main": "index.js",
|
"main": "index.mjs",
|
||||||
|
"type": "module",
|
||||||
"repository": "ssh://git@10.0.2.28:222/lino-authelia/bpi-stock-price-scraper.git",
|
"repository": "ssh://git@10.0.2.28:222/lino-authelia/bpi-stock-price-scraper.git",
|
||||||
"author": "Lino Silva <me@lino.cooking>",
|
"author": "Lino Silva <me@lino.cooking>",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
|
"scripts": {
|
||||||
|
"start": "node index.mjs",
|
||||||
|
"dev": "node --watch index.mjs"
|
||||||
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"axios": "^1.7.7",
|
|
||||||
"dotenv": "^16.4.5",
|
"dotenv": "^16.4.5",
|
||||||
|
"express": "^5.2.1",
|
||||||
"puppeteer": "^23.5.3"
|
"puppeteer": "^23.5.3"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
|
|||||||
Generated
+2002
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user