-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathMakefile
More file actions
executable file
·89 lines (76 loc) · 2.94 KB
/
Makefile
File metadata and controls
executable file
·89 lines (76 loc) · 2.94 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
.PHONY: init check-secrets up down restart logs psql-warehouse psql-meta clean nuke setup-gold help
DC = docker-compose
CONTAINER_META = pizza_metadata_db
CONTAINER_WAREHOUSE = pizza_warehouse_db
CONTAINER_SCHEDULER = pizza_airflow_scheduler
GOLD_SCHEMA_FILE = models/gold_schema_md.sql
help:
@echo "🛠️ Commands:"
@echo " make init - Setup folders and prompt for passwords"
@echo " make up - Build and start containers"
@echo " make setup-gold - 🏗️ Apply Gold Schema from $(GOLD_SCHEMA_FILE)"
@echo " make down - Stop containers"
@echo " make restart - Restart containers"
@echo " make logs - Tail Airflow Scheduler logs"
@echo " make psql-warehouse - Connect to Data Warehouse"
@echo " make psql-meta - Connect to Metadata DB"
@echo " make clean - Prune docker & clean checkpoints"
@echo " make nuke - Destroy everything including volumes!"
check-secrets:
@mkdir -p secrets
@if [ ! -s secrets/postgres_password.txt ]; then \
printf "🔑 Podaj hasło dla ROOT Postgresa (postgres_password.txt): "; \
read pass; \
echo "$$pass" > secrets/postgres_password.txt; \
fi
@if [ ! -s secrets/airflow_db_password.txt ]; then \
printf "🔑 Podaj hasło dla AIRFLOW DB (airflow_db_password.txt): "; \
read pass; \
echo "$$pass" > secrets/airflow_db_password.txt; \
fi
@if [ ! -s secrets/warehouse_password.txt ]; then \
printf "🔑 Podaj hasło dla WAREHOUSE DB (warehouse_password.txt): "; \
read pass; \
echo "$$pass" > secrets/warehouse_password.txt; \
fi
init: check-secrets
@echo "📂 Creating project structure..."
mkdir -p logs secrets models
mkdir -p data/bronze data/silver data/gold data/checkpoints data/archive
mkdir -p src/common src/generator src/pipelines/silver src/pipelines/gold
@echo "📜 Creating SQL init files if missing..."
@touch init_metadata.sql
@touch init_warehouse.sql
@echo "✅ Init complete."
setup-gold:
@echo "🏗️ Wdrażanie schematu GOLD z pliku $(GOLD_SCHEMA_FILE)..."
@if [ -f $(GOLD_SCHEMA_FILE) ]; then \
cat $(GOLD_SCHEMA_FILE) | docker exec -i $(CONTAINER_WAREHOUSE) psql -U warehouse_admin -d data_warehouse; \
echo "✅ Schemat wdrożony pomyślnie."; \
else \
echo "❌ Błąd: Nie znaleziono pliku $(GOLD_SCHEMA_FILE)"; \
fi
up: check-secrets
@echo "🐳 Starting containers..."
$(DC) up -d --build --remove-orphans
@echo "✅ System is running!"
down:
$(DC) down
restart: down up
logs:
$(DC) logs -f $(CONTAINER_SCHEDULER)
psql-warehouse:
@echo "🔌 Connecting to Data Warehouse..."
docker exec -it $(CONTAINER_WAREHOUSE) psql -U warehouse_admin -d data_warehouse
psql-meta:
@echo "🔌 Connecting to Metadata DB..."
docker exec -it $(CONTAINER_META) psql -U root_admin -d postgres
clean:
@echo "🧹 Cleaning up..."
docker system prune -f
rm -rf data/checkpoints/*
nuke:
@echo "NUKING EVERYTHING (Volumes included)..."
$(DC) down -v
rm -rf data/checkpoints/*
@echo "✅ Clean slate."