feat: Initial move to docker image
Build and publish / build (push) Failing after 2m37s

This commit is contained in:
Lino Silva
2026-04-10 17:24:44 +01:00
parent be8ff08f94
commit fe4c54ebcb
10 changed files with 3153 additions and 112 deletions
+12
View File
@@ -0,0 +1,12 @@
node_modules
npm-debug.log
.git
.gitignore
.env
.DS_Store
*.md
!README.md
.vscode
.idea
coverage
.eslintcache
+2 -2
View File
@@ -1,2 +1,2 @@
GHOSTFOLIO_SECURITY_TOKEN= # Optional: Port to run the server on (defaults to 3000)
GHOSTFOLIO_HOST= PORT=3000
+37 -19
View File
@@ -2,30 +2,48 @@ name: Build and publish
run-name: Build docker image and publish to registry run-name: Build docker image and publish to registry
on: on:
schedule:
- cron: '0 10,22 * * *'
push: push:
branches: branches:
- 'main' - 'main'
tags:
- 'v*'
jobs: jobs:
build: build:
runs-on: ubuntu-latest runs-on: ubuntu-latest
container:
image: ghcr.io/catthehacker/ubuntu:act-latest
steps: steps:
- run: apt update - name: Checkout code
- run: apt install -y libatk1.0-0 libatk-bridge2.0-0 libasound2t64 libgdk-pixbuf2.0-0 libgtk-3-0 libgbm-dev libnss3-dev libxss-dev uses: https://github.com/actions/checkout@v4
name: Prepare environment
- uses: https://github.com/actions/checkout@v4 - name: Set up Docker Buildx
- name: Use Node.js uses: https://github.com/docker/setup-buildx-action@v3
uses: https://github.com/actions/setup-node@v3
- name: Log in to Gitea Container Registry
uses: https://github.com/docker/login-action@v3
with: with:
node-version: '20.18' registry: 10.0.2.28:222
- run: npm i username: lino
name: Install dependencies password: ${{ secrets.REGISTRY_PASSWORD }}
- run: node index.mjs
name: Run app - name: Extract metadata
env: id: meta
GHOSTFOLIO_SECURITY_TOKEN: ${{ secrets.GHOSTFOLIO_SECURITY_TOKEN }} uses: https://github.com/docker/metadata-action@v5
GHOSTFOLIO_HOST: ${{ secrets.GHOSTFOLIO_HOST }} with:
images: 10.0.2.28:222/${{ gitea.repository }}
tags: |
type=ref,event=branch
type=ref,event=pr
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=sha,prefix={{branch}}-
type=raw,value=latest,enable={{is_default_branch}}
- name: Build and push Docker image
uses: https://github.com/docker/build-push-action@v5
with:
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
+36
View File
@@ -0,0 +1,36 @@
# Use Node.js LTS with Debian (needed for Puppeteer dependencies)
FROM node:20-bookworm-slim
# Install Chromium dependencies for Puppeteer
RUN apt-get update && apt-get install -y \
chromium \
fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-freefont-ttf \
libxss1 \
--no-install-recommends \
&& rm -rf /var/lib/apt/lists/*
# Set Puppeteer to use installed Chromium
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
# Create app directory
WORKDIR /usr/src/app
# Copy package files
COPY package*.json ./
# Install dependencies
RUN npm ci --only=production
# Copy application files
COPY . .
# Expose port
EXPOSE 3000
# Health check
HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \
CMD node -e "require('http').get('http://localhost:3000/health', (r) => {process.exit(r.statusCode === 200 ? 0 : 1)})"
# Run the application
CMD ["npm", "start"]
+87 -11
View File
@@ -1,21 +1,97 @@
# bpi-stock-price-scraper # bpi-stock-price-scraper
Extrair preço de PPRs para introduzir no Ghostfolio Web server API to fetch latest BPI PPR stock prices on demand.
# Utilização ## Features
Criar ficheiro `.env` a partir do `.env.example` - REST API endpoints to get latest prices for BPI Destino PPR funds
- Returns price data as JSON
- Dockerized for easy deployment
- Health check endpoint included
## Docker ## Available Symbols
``` - `BPIDEST2040` - BPI Destino PPR 2040
docker compose build - `BPIDEST2050` - BPI Destino PPR 2050
docker compose up
## API Endpoints
- `GET /` - API information and usage examples
- `GET /health` - Health check with available symbols
- `GET /price/:symbol` - Get latest price for a specific symbol
### Example Response
```json
{
"symbol": "BPIDEST2040",
"name": "BPI Destino PPR 2040",
"price": 7.73313,
"date": "2026-04-09",
"timestamp": "2026-04-10T16:11:25.850Z"
}
``` ```
## Node ## Usage
### Docker (Recommended)
Build and run with docker-compose:
```bash
docker compose up -d
```
Or build and run manually:
```bash
docker build -t bpi-stock-price-scraper .
docker run -p 3000:3000 bpi-stock-price-scraper
```
### Node.js
Install dependencies and run:
```bash
npm install
npm start
```
For development with auto-reload:
```bash
npm run dev
```
## Configuration
The server runs on port 3000 by default. You can change it via the `PORT` environment variable:
```bash
PORT=8080 npm start
```
Or in docker-compose.yml:
```yaml
environment:
- PORT=8080
ports:
- "8080:8080"
```
## Testing
Test the API:
```bash
# Get API info
curl http://localhost:3000/
# Check health
curl http://localhost:3000/health
# Get latest price
curl http://localhost:3000/price/BPIDEST2040
``` ```
yarn
node index.mjs
```
+16
View File
@@ -0,0 +1,16 @@
version: '3.8'
services:
bpi-scraper:
build: .
ports:
- "3000:3000"
environment:
- PORT=3000
restart: unless-stopped
healthcheck:
test: ["CMD", "node", "-e", "require('http').get('http://localhost:3000/health', (r) => {process.exit(r.statusCode === 200 ? 0 : 1)})"]
interval: 30s
timeout: 10s
retries: 3
start_period: 40s
+102 -77
View File
@@ -1,103 +1,128 @@
import puppeteer from "puppeteer"; import puppeteer from "puppeteer";
import process from "node:process"; import express from "express";
import axios from "axios";
import "dotenv/config"; import "dotenv/config";
const contas = { const app = express();
bpi2040: { const PORT = process.env.PORT || 3000;
// Symbol configuration
const symbols = {
BPIDEST2040: {
url: "https://www.bancobpi.pt/particulares/poupar-investir/ppr/bpi-destino-ppr-2040", url: "https://www.bancobpi.pt/particulares/poupar-investir/ppr/bpi-destino-ppr-2040",
ghostfolioName: "GF_BPI_Destino_PPR_2040", name: "BPI Destino PPR 2040",
}, },
bpi2050: { BPIDEST2050: {
url: "https://www.bancobpi.pt/particulares/poupar-investir/ppr/bpi-destino-ppr-2050", url: "https://www.bancobpi.pt/particulares/poupar-investir/ppr/bpi-destino-ppr-2050",
ghostfolioName: "GF_BPI_Destino_PPR_2050", name: "BPI Destino PPR 2050",
}, },
}; };
async function parseLogRocketBlogHome(conta) { // Function to scrape price for a given symbol
async function getLatestPrice(symbol) {
const config = symbols[symbol];
if (!config) {
throw new Error(`Invalid symbol: ${symbol}`);
}
// Launch the browser // Launch the browser
const browser = await puppeteer.launch({ const browser = await puppeteer.launch({
args: ["--no-sandbox"], args: ["--no-sandbox"],
timeout: 10000, timeout: 10000,
}); });
// Open a new tab try {
const page = await browser.newPage(); // Open a new tab
const page = await browser.newPage();
// Visit the page and wait until network connections are completed // Visit the page and wait until network connections are completed
await page.goto(conta.url, { waitUntil: "networkidle2" }); await page.goto(config.url, { waitUntil: "networkidle2" });
// Interact with the DOM to retrieve the titles // Interact with the DOM to retrieve the price and date
const [price, date] = await page.evaluate(() => { const [price, date] = await page.evaluate(() => {
const spanTags = document.getElementsByTagName("span"); const spanTags = document.getElementsByTagName("span");
const priceSearchText = "ÚLTIMA COTAÇÃO:"; const priceSearchText = "ÚLTIMA COTAÇÃO:";
const dateSearchText = "DATA COTAÇÃO:"; const dateSearchText = "DATA COTAÇÃO:";
let priceElementFound; let priceElementFound;
let dateElementFound; let dateElementFound;
for (let i = 0; i < spanTags.length; i++) { for (let i = 0; i < spanTags.length; i++) {
if (spanTags[i].textContent.trim() == priceSearchText) { if (spanTags[i].textContent.trim() == priceSearchText) {
priceElementFound = spanTags[i]; priceElementFound = spanTags[i];
console.log(`Found ${priceSearchText}.`); if (priceElementFound && dateElementFound) break;
if (priceElementFound && dateElementFound) break; }
if (spanTags[i].textContent.trim() == dateSearchText) {
dateElementFound = spanTags[i];
if (priceElementFound && dateElementFound) break;
}
} }
if (spanTags[i].textContent.trim() == dateSearchText) {
dateElementFound = spanTags[i]; if (!priceElementFound || !dateElementFound) {
console.log(`Found ${dateSearchText}.`); throw new Error("Could not find price or date elements");
if (priceElementFound && dateElementFound) break;
} }
}
return [ return [
priceElementFound.nextSibling.innerHTML, priceElementFound.nextSibling.innerHTML,
dateElementFound.nextSibling.innerHTML, dateElementFound.nextSibling.innerHTML,
]; ];
}); });
// Don't forget to close the browser instance to clean up the memory const marketPrice = parseFloat(
await browser.close(); price.replace("€", "").trim().replace(",", "."),
);
const marketPrice = parseFloat( return {
price.replace("€", "").trim().replace(",", "."), symbol,
); name: config.name,
price: marketPrice,
// Print the results date: date.trim(),
console.log(decodeURIComponent(conta.ghostfolioName)); timestamp: new Date().toISOString(),
console.log(`Current price: € ${marketPrice}`); };
console.log(`Date: ${date}`); } finally {
// Always close the browser
const bearerTokenResponse = await axios.post( await browser.close();
`${process.env.GHOSTFOLIO_HOST}/api/v1/auth/anonymous`, }
{
accessToken: process.env.GHOSTFOLIO_SECURITY_TOKEN,
},
);
const response = await axios
.post(
`${process.env.GHOSTFOLIO_HOST}/api/v1/market-data/MANUAL/${conta.ghostfolioName}`,
{
marketData: [
{
date,
marketPrice,
},
],
},
{
headers: {
Authorization: `Bearer ${bearerTokenResponse.data.authToken}`,
},
},
)
.catch(console.error);
console.log(response.status);
} }
await Promise.all([ // API endpoint to get latest price for a symbol
parseLogRocketBlogHome(contas.bpi2040), app.get("/price/:symbol", async (req, res) => {
parseLogRocketBlogHome(contas.bpi2050), const { symbol } = req.params;
]);
process.exit(0); try {
console.log(`Fetching price for symbol: ${symbol}`);
const priceData = await getLatestPrice(symbol);
console.log(`Success: ${symbol} - €${priceData.price}`);
res.json(priceData);
} catch (error) {
console.error(`Error fetching price for ${symbol}:`, error.message);
res.status(400).json({
error: error.message,
symbol,
});
}
});
// Health check endpoint
app.get("/health", (req, res) => {
res.json({ status: "ok", availableSymbols: Object.keys(symbols) });
});
// Root endpoint with usage instructions
app.get("/", (req, res) => {
res.json({
message: "BPI Stock Price Scraper API",
endpoints: {
health: "/health",
price: "/price/:symbol",
},
availableSymbols: Object.keys(symbols),
example: `/price/BPIDEST2040`,
});
});
// Start the server
app.listen(PORT, () => {
console.log(`Server running on http://localhost:${PORT}`);
console.log(`Available symbols: ${Object.keys(symbols).join(", ")}`);
console.log(`Example: http://localhost:${PORT}/price/BPIDEST2040`);
});
+852 -1
View File
File diff suppressed because it is too large Load Diff
+7 -2
View File
@@ -1,13 +1,18 @@
{ {
"name": "bpi-stock-price-scraper", "name": "bpi-stock-price-scraper",
"version": "1.0.0", "version": "1.0.0",
"main": "index.js", "main": "index.mjs",
"type": "module",
"repository": "ssh://git@10.0.2.28:222/lino-authelia/bpi-stock-price-scraper.git", "repository": "ssh://git@10.0.2.28:222/lino-authelia/bpi-stock-price-scraper.git",
"author": "Lino Silva <me@lino.cooking>", "author": "Lino Silva <me@lino.cooking>",
"license": "MIT", "license": "MIT",
"scripts": {
"start": "node index.mjs",
"dev": "node --watch index.mjs"
},
"dependencies": { "dependencies": {
"axios": "^1.7.7",
"dotenv": "^16.4.5", "dotenv": "^16.4.5",
"express": "^5.2.1",
"puppeteer": "^23.5.3" "puppeteer": "^23.5.3"
}, },
"devDependencies": { "devDependencies": {
+2002
View File
File diff suppressed because it is too large Load Diff