Skip to content

Commit df5e0d0

Browse files
authored
Update config and paths to deploy services with docker compose (#696) (#697)
* Fix docker tiler volume for staging and production * Fix taginfo data processor image * Update readme * update * Update configs * Update deploy scripts * Update scripts * Display template * Remove tilerdb adn imposm from base * Add network tiler-db * Restart taginfo web after data has been proceed * Update overpass docker image
1 parent 4e8337b commit df5e0d0

File tree

12 files changed

+169
-283
lines changed

12 files changed

+169
-283
lines changed

hetzner/deploy.sh

Lines changed: 29 additions & 128 deletions
Original file line numberDiff line numberDiff line change
@@ -1,150 +1,51 @@
11
#!/bin/bash
22
set -e
33

4-
# Parse --yes / -y so script can run non-interactively
54
AUTO_YES=false
65
ARGS=()
76
for a in "$@"; do
8-
if [[ "$a" == "--yes" || "$a" == "-y" ]]; then
9-
AUTO_YES=true
10-
else
11-
ARGS+=("$a")
12-
fi
7+
[[ "$a" == "--yes" || "$a" == "-y" ]] && AUTO_YES=true || ARGS+=("$a")
138
done
149

1510
ACTION=${ARGS[0]}
1611
SERVICE=${ARGS[1]}
17-
ENVIRONMENT=${ARGS[2]:-staging}
12+
ENVIRONMENT=${ARGS[2]}
1813

19-
# Check if first arg is an action (start/stop/restart)
20-
if [ "$ACTION" = "start" ] || [ "$ACTION" = "stop" ] || [ "$ACTION" = "restart" ]; then
21-
# First arg is an action, so SERVICE is $2
22-
if [ -z "$SERVICE" ]; then
23-
echo "Usage: $0 [--yes|-y] start|stop|restart <service> [staging|production]"
24-
echo "Example: $0 start taginfo staging"
25-
exit 1
26-
fi
27-
fi
28-
29-
if [ -z "$SERVICE" ]; then
30-
echo "Usage: $0 [--yes|-y] [start|stop|restart] <service> [staging|production]"
31-
echo " --yes, -y Skip confirmation prompts"
32-
echo "Examples:"
33-
echo " $0 taginfo staging # Start service"
34-
echo " $0 --yes start taginfo production # Start without prompting"
35-
echo " $0 stop taginfo staging # Stop service"
36-
echo " $0 restart taginfo staging # Restart service"
14+
if [ -z "$SERVICE" ] || [ -z "$ENVIRONMENT" ]; then
15+
echo "Usage: $0 [--yes|-y] start|stop|restart <service> staging|production"
16+
echo "Example: $0 start taginfo staging"
3717
exit 1
3818
fi
3919

40-
SERVICE_DIR="$(cd "$(dirname "$0")" && pwd)/$SERVICE"
41-
BASE_FILE="$SERVICE_DIR/$SERVICE.base.yml"
42-
ENV_FILE="$SERVICE_DIR/$SERVICE.$ENVIRONMENT.yml"
43-
44-
# Load environment variables from .env.traefik
4520
HETZNER_DIR="$(cd "$(dirname "$0")" && pwd)"
46-
if [ -f "$HETZNER_DIR/.env.traefik" ]; then
47-
export $(grep -v '^#' "$HETZNER_DIR/.env.traefik" | xargs)
48-
fi
21+
BASE_FILE="$HETZNER_DIR/$SERVICE/$SERVICE.base.yml"
22+
ENV_FILE="$HETZNER_DIR/$SERVICE/$SERVICE.$ENVIRONMENT.yml"
4923

50-
# For staging, only use base file. For production, use base + environment file
51-
if [ "$ENVIRONMENT" = "staging" ]; then
52-
COMPOSE_CMD="docker compose -f $BASE_FILE"
24+
[ -f "$HETZNER_DIR/.env.traefik" ] && export $(grep -v '^#' "$HETZNER_DIR/.env.traefik" | xargs)
25+
26+
if [ -f "$ENV_FILE" ]; then
27+
COMPOSE="docker compose -f $BASE_FILE -f $ENV_FILE"
5328
else
54-
COMPOSE_CMD="docker compose -f $BASE_FILE -f $ENV_FILE"
29+
COMPOSE="docker compose -f $BASE_FILE"
5530
fi
5631

57-
echo "================================================"
58-
echo "Action: $ACTION"
59-
echo "Service: $SERVICE"
60-
echo "Environment: $ENVIRONMENT"
61-
echo "Command: $COMPOSE_CMD"
62-
echo "================================================"
32+
echo "==> $ACTION $SERVICE ($ENVIRONMENT)"
33+
34+
# Confirm before start/restart (skip with --yes)
35+
if [[ "$ACTION" == "start" || "$ACTION" == "restart" ]] && [ "$AUTO_YES" != "true" ]; then
36+
[ "$ENVIRONMENT" = "production" ] && echo "WARNING: Deploying to PRODUCTION"
37+
echo ""
38+
$COMPOSE config
39+
echo ""
40+
read -p "Continue? (yes/no): " confirm
41+
[ "$confirm" != "yes" ] && echo "Cancelled." && exit 0
42+
fi
6343

6444
case "$ACTION" in
65-
start)
66-
# Show merged configuration before deploying
67-
echo ""
68-
echo "Preview of merged configuration:"
69-
echo "================================================"
70-
71-
# Try to use colored output if available
72-
if command -v bat &> /dev/null; then
73-
$COMPOSE_CMD config | bat --language yaml --style=plain
74-
elif command -v pygmentize &> /dev/null; then
75-
$COMPOSE_CMD config | pygmentize -l yaml
76-
elif command -v highlight &> /dev/null; then
77-
$COMPOSE_CMD config | highlight --out-format=ansi --syntax=yaml
78-
else
79-
# Fallback: use basic colors with grep/awk
80-
$COMPOSE_CMD config | while IFS= read -r line; do
81-
if [[ "$line" =~ ^[[:space:]]*# ]]; then
82-
# Comments in gray
83-
echo -e "\033[0;90m$line\033[0m"
84-
elif [[ "$line" =~ ^[[:space:]]*[a-zA-Z_][a-zA-Z0-9_]*: ]]; then
85-
# Keys in yellow
86-
key=$(echo "$line" | sed 's/:.*//')
87-
rest=$(echo "$line" | sed 's/^[^:]*://')
88-
echo -e "\033[0;33m$key\033[0m:\033[0;36m$rest\033[0m"
89-
elif [[ "$line" =~ ^[[:space:]]*- ]]; then
90-
# List items in cyan
91-
echo -e "\033[0;36m$line\033[0m"
92-
else
93-
echo "$line"
94-
fi
95-
done
96-
fi
97-
98-
echo "================================================"
99-
echo ""
100-
101-
# Ask for confirmation (skip if --yes/-y)
102-
if [ "$AUTO_YES" != "true" ]; then
103-
if [ "$ENVIRONMENT" = "production" ]; then
104-
echo "⚠️ WARNING: You are about to deploy to PRODUCTION"
105-
echo ""
106-
read -p "Do you want to continue? (yes/no): " confirm
107-
if [ "$confirm" != "yes" ]; then
108-
echo "Deployment cancelled."
109-
exit 0
110-
fi
111-
else
112-
read -p "Do you want to continue with deployment? (yes/no): " confirm
113-
if [ "$confirm" != "yes" ]; then
114-
echo "Deployment cancelled."
115-
exit 0
116-
fi
117-
fi
118-
fi
119-
120-
echo ""
121-
echo "Starting deployment..."
122-
$COMPOSE_CMD up -d
123-
echo ""
124-
echo "✓ Service started: $SERVICE ($ENVIRONMENT)"
125-
echo ""
126-
echo "================================================"
127-
echo "Useful commands:"
128-
echo "================================================"
129-
echo ""
130-
echo "# List running containers for this service:"
131-
echo "docker ps | grep ${SERVICE}"
132-
echo "docker exec -it ${SERVICE}_${ENVIRONMENT} bash"
133-
echo ""
134-
echo "# View logs:"
135-
echo "$COMPOSE_CMD logs -f"
136-
echo ""
137-
;;
138-
stop)
139-
$COMPOSE_CMD down
140-
echo "Service stopped: $SERVICE ($ENVIRONMENT)"
141-
;;
142-
restart)
143-
$COMPOSE_CMD restart
144-
echo "Service restarted: $SERVICE ($ENVIRONMENT)"
145-
;;
146-
*)
147-
echo "Unknown action: $ACTION"
148-
exit 1
149-
;;
45+
start) $COMPOSE up -d ;;
46+
stop) $COMPOSE down ;;
47+
restart) $COMPOSE up -d --force-recreate ;;
48+
*) echo "Unknown action: $ACTION"; exit 1 ;;
15049
esac
50+
51+
echo "Done: $ACTION $SERVICE ($ENVIRONMENT)"

hetzner/osmcha/osmcha.base.yml

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,13 @@ services:
33
image: postgis/postgis:17-3.5
44
container_name: osmcha-db
55
restart: always
6-
env_file: ./.env.osmcha
6+
env_file:
7+
- ./.env.osmcha
78
volumes:
89
- osmcha_db_data:/var/lib/postgresql/data
910
- ./data_backup:/data_backup
1011
healthcheck:
11-
test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
12+
test: ["CMD-SHELL", "pg_isready"]
1213
interval: 5s
1314
timeout: 5s
1415
retries: 10
@@ -30,7 +31,8 @@ services:
3031
python manage.py migrate &&
3132
python manage.py collectstatic --noinput
3233
"
33-
env_file: ./.env.osmcha
34+
env_file:
35+
- ./.env.osmcha
3436
volumes:
3537
- staticfiles:/app/staticfiles
3638
depends_on:
@@ -47,7 +49,8 @@ services:
4749
command: gunicorn config.wsgi -b 0.0.0.0:5000 --access-logfile - --timeout 120 --workers 4 --threads 16
4850
# ports:
4951
# - "5100:5000"
50-
env_file: ./.env.osmcha
52+
env_file:
53+
- ./.env.osmcha
5154
volumes:
5255
- staticfiles:/staticfiles
5356
depends_on:
@@ -76,7 +79,8 @@ services:
7679
osmcha-cron:
7780
container_name: osmcha-cron
7881
image: ghcr.io/openhistoricalmap/osmcha-django:1bd58e1
79-
env_file: ./.env.osmcha
82+
env_file:
83+
- ./.env.osmcha
8084
depends_on:
8185
osmcha-db:
8286
condition: service_healthy

hetzner/osmcha/osmcha.production.yml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,11 @@ services:
1515
volumes:
1616
- ohmx_db:/data
1717
- ${HOME}/.aws:/root/.aws:ro
18-
- /production/services/images/ohmx-adiff-builder/config.sh:/app/config.sh
19-
- /production/services/images/ohmx-adiff-builder/functions.sh:/app/functions.sh
20-
- /production/services/images/ohmx-adiff-builder/start.sh:/app/start.sh
21-
- /production/services/images/ohmx-adiff-builder/process_min_range.sh:/app/process_min_range.sh
22-
- /production/services/images/ohmx-adiff-builder/update.sh:/app/update.sh
18+
- ${PWD}/images/ohmx-adiff-builder/config.sh:/app/config.sh
19+
- ${PWD}/images/ohmx-adiff-builder/functions.sh:/app/functions.sh
20+
- ${PWD}/images/ohmx-adiff-builder/start.sh:/app/start.sh
21+
- ${PWD}/images/ohmx-adiff-builder/process_min_range.sh:/app/process_min_range.sh
22+
- ${PWD}/images/ohmx-adiff-builder/update.sh:/app/update.sh
2323
networks:
2424
- ohm_network
2525
cpus: '8.0'

hetzner/overpass/overpass.base.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
services:
22
overpass:
3-
image: developmentseed/osmseed-overpass-api:0.1.0-0.dev.git.956.h49d677b
3+
image: ghcr.io/osm-seed/overpass-api:0.1.0-0.dev.git.985.h3b10440
44
container_name: overpass
55
# ports:
66
# - '8085:80'

hetzner/start_all.sh

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,20 @@ set -e
33

44
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
55

6-
ENVIRONMENT=${ENVIRONMENT:-staging}
6+
ENVIRONMENT=${ENVIRONMENT}
7+
8+
# Validate ENVIRONMENT is set and is either staging or production
9+
if [ -z "$ENVIRONMENT" ]; then
10+
echo ""
11+
echo "Need to set ENVIRONMENT variable:"
12+
echo " export ENVIRONMENT=staging"
13+
echo " export ENVIRONMENT=production"
14+
exit 1
15+
fi
16+
17+
echo "########################## ENVIRONMENT -> $ENVIRONMENT ##########################"
718

8-
# Load environment variables from .env.traefik
19+
# Load environment variables from .env.traefik, make sure the domain is set
920
source "$SCRIPT_DIR/.env.traefik"
1021
echo "########################## OHM_DOMAIN -> $OHM_DOMAIN ##########################"
1122

@@ -38,3 +49,8 @@ if [ "$ENVIRONMENT" = "staging" ]; then
3849
docker stop tiler_db
3950
docker stop tiler_imposm
4051
fi
52+
53+
## In production we need to clean the tiler cache
54+
docker stop tiler_s3_cleaner
55+
## clean tiler cache
56+
# docker compose -f hetzner/tiler/tiler.base.yml -f hetzner/tiler/tiler.production.yml run tiler_s3_cleaner tiler-cache-cleaner clean_by_prefix

hetzner/taginfo/taginfo.base.yml

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,9 @@
11
services:
22
taginfo_web:
33
container_name: taginfo_web
4-
image: ghcr.io/osm-seed/taginfo-web:0.1.0-0.dev.git.983.h32750c8
5-
volumes:
6-
- ${HOME}/data/staging/taginfo_data:/usr/src/app/data
4+
image: ghcr.io/osm-seed/taginfo-web:0.1.0-0.dev.git.986.h0fae372
75
env_file:
8-
- ./.env.sample
6+
- ./.env.taginfo
97
restart: always
108
networks:
119
- ohm_network
Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,33 @@
11
services:
2+
23
taginfo_data:
4+
container_name: taginfo_data
5+
image: ghcr.io/osm-seed/taginfo:0.1.0-0.dev.git.984.hb289ff1
36
volumes:
7+
- /var/run/docker.sock:/var/run/docker.sock
48
- ${HOME}/data/production/taginfo_data:/usr/src/app/data
59
- ${HOME}/data/production/taginfo_data_planet:/osm/planet/var
610
environment:
711
- AWS_S3_BUCKET=planet.openhistoricalmap.org
12+
- DOCKER_CONFIG_ENVIRONMENT=production
813
command:
914
- /bin/bash
1015
- -lc
1116
- |
1217
set -ex
13-
# Run every 3 days at 3:00 AM
14-
echo "0 3 1-31/3 * * /usr/src/app/start.sh >> /var/log/taginfo-cron.log 2>&1" > /tmp/cron_logs
15-
crontab /tmp/cron_logs
18+
echo "#!/bin/bash
19+
/usr/src/app/start.sh
20+
sleep 7200
21+
# restart web container
22+
curl -X POST --unix-socket /var/run/docker.sock http://v1.41/containers/taginfo_web/restart" > /run_task.sh
23+
chmod +x /run_task.sh
24+
25+
# 2. Set cron: Run every 3 days at 3:00 AM
26+
echo "0 3 */3 * * /run_task.sh >> /var/log/taginfo-cron.log 2>&1" | crontab -
27+
1628
touch /var/log/taginfo-cron.log
1729
cron -f
18-
19-
taginfo_web:
20-
volumes:
21-
- ${HOME}/data/production/taginfo_data_proceced:/usr/src/app/data
30+
env_file:
31+
- ./.env.taginfo
32+
networks:
33+
- ohm_network

0 commit comments

Comments
 (0)