feat: Initial move to docker image
Build and publish / build (push) Failing after 2m37s

This commit is contained in:
Lino Silva
2026-04-10 17:24:44 +01:00
parent be8ff08f94
commit fe4c54ebcb
10 changed files with 3153 additions and 112 deletions
+12
View File
@@ -0,0 +1,12 @@
node_modules
npm-debug.log
.git
.gitignore
.env
.DS_Store
*.md
!README.md
.vscode
.idea
coverage
.eslintcache
+2 -2
View File
@@ -1,2 +1,2 @@
GHOSTFOLIO_SECURITY_TOKEN=
GHOSTFOLIO_HOST=
# Optional: Port to run the server on (defaults to 3000)
PORT=3000
+37 -19
View File
@@ -2,30 +2,48 @@ name: Build and publish
run-name: Build docker image and publish to registry
on:
schedule:
- cron: '0 10,22 * * *'
push:
branches:
- 'main'
- 'main'
tags:
- 'v*'
jobs:
build:
runs-on: ubuntu-latest
container:
image: ghcr.io/catthehacker/ubuntu:act-latest
steps:
- run: apt update
- run: apt install -y libatk1.0-0 libatk-bridge2.0-0 libasound2t64 libgdk-pixbuf2.0-0 libgtk-3-0 libgbm-dev libnss3-dev libxss-dev
name: Prepare environment
- uses: https://github.com/actions/checkout@v4
- name: Use Node.js
uses: https://github.com/actions/setup-node@v3
- name: Checkout code
uses: https://github.com/actions/checkout@v4
- name: Set up Docker Buildx
uses: https://github.com/docker/setup-buildx-action@v3
- name: Log in to Gitea Container Registry
uses: https://github.com/docker/login-action@v3
with:
node-version: '20.18'
- run: npm i
name: Install dependencies
- run: node index.mjs
name: Run app
env:
GHOSTFOLIO_SECURITY_TOKEN: ${{ secrets.GHOSTFOLIO_SECURITY_TOKEN }}
GHOSTFOLIO_HOST: ${{ secrets.GHOSTFOLIO_HOST }}
registry: 10.0.2.28:222
username: lino
password: ${{ secrets.REGISTRY_PASSWORD }}
- name: Extract metadata
id: meta
uses: https://github.com/docker/metadata-action@v5
with:
images: 10.0.2.28:222/${{ gitea.repository }}
tags: |
type=ref,event=branch
type=ref,event=pr
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=sha,prefix={{branch}}-
type=raw,value=latest,enable={{is_default_branch}}
- name: Build and push Docker image
uses: https://github.com/docker/build-push-action@v5
with:
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
+36
View File
@@ -0,0 +1,36 @@
# Use Node.js LTS with Debian (needed for Puppeteer dependencies)
FROM node:20-bookworm-slim
# Install Chromium dependencies for Puppeteer
RUN apt-get update && apt-get install -y \
chromium \
fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-freefont-ttf \
libxss1 \
--no-install-recommends \
&& rm -rf /var/lib/apt/lists/*
# Set Puppeteer to use installed Chromium
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
# Create app directory
WORKDIR /usr/src/app
# Copy package files
COPY package*.json ./
# Install dependencies
RUN npm ci --only=production
# Copy application files
COPY . .
# Expose port
EXPOSE 3000
# Health check
HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \
CMD node -e "require('http').get('http://localhost:3000/health', (r) => {process.exit(r.statusCode === 200 ? 0 : 1)})"
# Run the application
CMD ["npm", "start"]
+87 -11
View File
@@ -1,21 +1,97 @@
# bpi-stock-price-scraper
Extrair preço de PPRs para introduzir no Ghostfolio
Web server API to fetch latest BPI PPR stock prices on demand.
# Utilização
## Features
Criar ficheiro `.env` a partir do `.env.example`
- REST API endpoints to get latest prices for BPI Destino PPR funds
- Returns price data as JSON
- Dockerized for easy deployment
- Health check endpoint included
## Docker
## Available Symbols
```
docker compose build
docker compose up
- `BPIDEST2040` - BPI Destino PPR 2040
- `BPIDEST2050` - BPI Destino PPR 2050
## API Endpoints
- `GET /` - API information and usage examples
- `GET /health` - Health check with available symbols
- `GET /price/:symbol` - Get latest price for a specific symbol
### Example Response
```json
{
"symbol": "BPIDEST2040",
"name": "BPI Destino PPR 2040",
"price": 7.73313,
"date": "2026-04-09",
"timestamp": "2026-04-10T16:11:25.850Z"
}
```
## Node
## Usage
### Docker (Recommended)
Build and run with docker-compose:
```bash
docker compose up -d
```
Or build and run manually:
```bash
docker build -t bpi-stock-price-scraper .
docker run -p 3000:3000 bpi-stock-price-scraper
```
### Node.js
Install dependencies and run:
```bash
npm install
npm start
```
For development with auto-reload:
```bash
npm run dev
```
## Configuration
The server runs on port 3000 by default. You can change it via the `PORT` environment variable:
```bash
PORT=8080 npm start
```
Or in docker-compose.yml:
```yaml
environment:
- PORT=8080
ports:
- "8080:8080"
```
## Testing
Test the API:
```bash
# Get API info
curl http://localhost:3000/
# Check health
curl http://localhost:3000/health
# Get latest price
curl http://localhost:3000/price/BPIDEST2040
```
yarn
node index.mjs
```
+16
View File
@@ -0,0 +1,16 @@
version: '3.8'
services:
bpi-scraper:
build: .
ports:
- "3000:3000"
environment:
- PORT=3000
restart: unless-stopped
healthcheck:
test: ["CMD", "node", "-e", "require('http').get('http://localhost:3000/health', (r) => {process.exit(r.statusCode === 200 ? 0 : 1)})"]
interval: 30s
timeout: 10s
retries: 3
start_period: 40s
+102 -77
View File
@@ -1,103 +1,128 @@
import puppeteer from "puppeteer";
import process from "node:process";
import axios from "axios";
import express from "express";
import "dotenv/config";
const contas = {
bpi2040: {
const app = express();
const PORT = process.env.PORT || 3000;
// Symbol configuration
const symbols = {
BPIDEST2040: {
url: "https://www.bancobpi.pt/particulares/poupar-investir/ppr/bpi-destino-ppr-2040",
ghostfolioName: "GF_BPI_Destino_PPR_2040",
name: "BPI Destino PPR 2040",
},
bpi2050: {
BPIDEST2050: {
url: "https://www.bancobpi.pt/particulares/poupar-investir/ppr/bpi-destino-ppr-2050",
ghostfolioName: "GF_BPI_Destino_PPR_2050",
name: "BPI Destino PPR 2050",
},
};
async function parseLogRocketBlogHome(conta) {
// Function to scrape price for a given symbol
async function getLatestPrice(symbol) {
const config = symbols[symbol];
if (!config) {
throw new Error(`Invalid symbol: ${symbol}`);
}
// Launch the browser
const browser = await puppeteer.launch({
args: ["--no-sandbox"],
timeout: 10000,
});
// Open a new tab
const page = await browser.newPage();
try {
// Open a new tab
const page = await browser.newPage();
// Visit the page and wait until network connections are completed
await page.goto(conta.url, { waitUntil: "networkidle2" });
// Visit the page and wait until network connections are completed
await page.goto(config.url, { waitUntil: "networkidle2" });
// Interact with the DOM to retrieve the titles
const [price, date] = await page.evaluate(() => {
const spanTags = document.getElementsByTagName("span");
const priceSearchText = "ÚLTIMA COTAÇÃO:";
const dateSearchText = "DATA COTAÇÃO:";
let priceElementFound;
let dateElementFound;
// Interact with the DOM to retrieve the price and date
const [price, date] = await page.evaluate(() => {
const spanTags = document.getElementsByTagName("span");
const priceSearchText = "ÚLTIMA COTAÇÃO:";
const dateSearchText = "DATA COTAÇÃO:";
let priceElementFound;
let dateElementFound;
for (let i = 0; i < spanTags.length; i++) {
if (spanTags[i].textContent.trim() == priceSearchText) {
priceElementFound = spanTags[i];
console.log(`Found ${priceSearchText}.`);
if (priceElementFound && dateElementFound) break;
for (let i = 0; i < spanTags.length; i++) {
if (spanTags[i].textContent.trim() == priceSearchText) {
priceElementFound = spanTags[i];
if (priceElementFound && dateElementFound) break;
}
if (spanTags[i].textContent.trim() == dateSearchText) {
dateElementFound = spanTags[i];
if (priceElementFound && dateElementFound) break;
}
}
if (spanTags[i].textContent.trim() == dateSearchText) {
dateElementFound = spanTags[i];
console.log(`Found ${dateSearchText}.`);
if (priceElementFound && dateElementFound) break;
if (!priceElementFound || !dateElementFound) {
throw new Error("Could not find price or date elements");
}
}
return [
priceElementFound.nextSibling.innerHTML,
dateElementFound.nextSibling.innerHTML,
];
});
return [
priceElementFound.nextSibling.innerHTML,
dateElementFound.nextSibling.innerHTML,
];
});
// Don't forget to close the browser instance to clean up the memory
await browser.close();
const marketPrice = parseFloat(
price.replace("€", "").trim().replace(",", "."),
);
const marketPrice = parseFloat(
price.replace("€", "").trim().replace(",", "."),
);
// Print the results
console.log(decodeURIComponent(conta.ghostfolioName));
console.log(`Current price: € ${marketPrice}`);
console.log(`Date: ${date}`);
const bearerTokenResponse = await axios.post(
`${process.env.GHOSTFOLIO_HOST}/api/v1/auth/anonymous`,
{
accessToken: process.env.GHOSTFOLIO_SECURITY_TOKEN,
},
);
const response = await axios
.post(
`${process.env.GHOSTFOLIO_HOST}/api/v1/market-data/MANUAL/${conta.ghostfolioName}`,
{
marketData: [
{
date,
marketPrice,
},
],
},
{
headers: {
Authorization: `Bearer ${bearerTokenResponse.data.authToken}`,
},
},
)
.catch(console.error);
console.log(response.status);
return {
symbol,
name: config.name,
price: marketPrice,
date: date.trim(),
timestamp: new Date().toISOString(),
};
} finally {
// Always close the browser
await browser.close();
}
}
await Promise.all([
parseLogRocketBlogHome(contas.bpi2040),
parseLogRocketBlogHome(contas.bpi2050),
]);
// API endpoint to get latest price for a symbol
app.get("/price/:symbol", async (req, res) => {
const { symbol } = req.params;
process.exit(0);
try {
console.log(`Fetching price for symbol: ${symbol}`);
const priceData = await getLatestPrice(symbol);
console.log(`Success: ${symbol} - €${priceData.price}`);
res.json(priceData);
} catch (error) {
console.error(`Error fetching price for ${symbol}:`, error.message);
res.status(400).json({
error: error.message,
symbol,
});
}
});
// Health check endpoint
app.get("/health", (req, res) => {
res.json({ status: "ok", availableSymbols: Object.keys(symbols) });
});
// Root endpoint with usage instructions
app.get("/", (req, res) => {
res.json({
message: "BPI Stock Price Scraper API",
endpoints: {
health: "/health",
price: "/price/:symbol",
},
availableSymbols: Object.keys(symbols),
example: `/price/BPIDEST2040`,
});
});
// Start the server
app.listen(PORT, () => {
console.log(`Server running on http://localhost:${PORT}`);
console.log(`Available symbols: ${Object.keys(symbols).join(", ")}`);
console.log(`Example: http://localhost:${PORT}/price/BPIDEST2040`);
});
+852 -1
View File
File diff suppressed because it is too large Load Diff
+7 -2
View File
@@ -1,13 +1,18 @@
{
"name": "bpi-stock-price-scraper",
"version": "1.0.0",
"main": "index.js",
"main": "index.mjs",
"type": "module",
"repository": "ssh://git@10.0.2.28:222/lino-authelia/bpi-stock-price-scraper.git",
"author": "Lino Silva <me@lino.cooking>",
"license": "MIT",
"scripts": {
"start": "node index.mjs",
"dev": "node --watch index.mjs"
},
"dependencies": {
"axios": "^1.7.7",
"dotenv": "^16.4.5",
"express": "^5.2.1",
"puppeteer": "^23.5.3"
},
"devDependencies": {
+2002
View File
File diff suppressed because it is too large Load Diff