Compare commits

..

No commits in common. "master" and "v2.1.0" have entirely different histories.

10 changed files with 76 additions and 199 deletions

View File

@ -1,32 +0,0 @@
name: Build docker image
run-name: ${{ gitea.actor }} is runs ci pipeline
on: [ push ]
jobs:
push:
runs-on: ubuntu-latest
if: gitea.ref == 'refs/heads/master'
steps:
- uses: https://github.com/actions/checkout@v4
- name: Log in to the Container registry
uses: docker/login-action@v2
with:
registry: git.whimsoc.dev/${{gitea.repository_owner}}/openstreetmap-tile-server
username: ${{ gitea.repository_owner }}
password: ${{ secrets.PAT }}
# - name: Set up Docker Buildx
# uses: docker/setup-buildx-action@v3
# - name: Build and push Docker image
# uses: https://github.com/docker/build-push-action@v6
# with:
# context: .
# file: ./Dockerfile
# push: true
# tags: "git.whimsoc.dev/${{gitea.repository_owner}}/openstreetmap-tile-server:${{gitea.sha}},git.whimsoc.dev/${{gitea.repository_owner}}/openstreetmap-tile-server:latest"
- name: Build and push
run: |
TODAY=$(date +'%Y-%m-%d')
docker build -t git.whimsoc.dev/${{gitea.repository_owner}}/openstreetmap-tile-server:${TODAY} -t git.whimsoc.dev/${{gitea.repository_owner}}/openstreetmap-tile-server:latest .
docker push git.whimsoc.dev/${{gitea.repository_owner}}/openstreetmap-tile-server:${TODAY}
docker push git.whimsoc.dev/${{gitea.repository_owner}}/openstreetmap-tile-server:latest

View File

@ -5,10 +5,10 @@ on:
branches:
- master
tags:
- 'v[0-9]+.[0-9]+.[0-9]+'
# pull_request:
# branches:
# - master
- 'v*'
pull_request:
branches:
- master
# cancel outdated jobs for the same reference
concurrency:
@ -21,65 +21,49 @@ env:
jobs:
build:
strategy:
matrix:
include:
- arch : amd64
mode : build-and-test
- arch : arm64
variant : v8
mode : build-only
test:
runs-on: ubuntu-latest
env:
VOLUME : osm-db
CONTAINER : osm-www
MOUNT : /data/database/
PLATFORM : linux/${{ matrix.arch }}${{ (matrix.variant != '' && format('/{0}', matrix.variant)) || '' }}
steps:
-
name: Checkout
uses: actions/checkout@v3
-
name: Set up QEMU
uses: docker/setup-qemu-action@v2
with:
platforms: ${{ matrix.arch }}
uses: docker/setup-qemu-action@v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v1
-
name: Environment
run : |
echo IMAGE=$(echo ${{ env.IMAGE }} | tr '[:upper:]' '[:lower:]') >>$GITHUB_ENV
-
name: Docker build
uses: docker/build-push-action@v3
uses: docker/build-push-action@v2
with:
pull : true
load : ${{ matrix.mode == 'build-and-test' }}
platforms : ${{ env.PLATFORM }}
load : true
context : .
file : ./Dockerfile
tags : ${{env.IMAGE}}:${{env.TAG}}
cache-from : type=gha,scope=${{ github.workflow }}:${{ env.PLATFORM }}
cache-to : type=gha,scope=${{ github.workflow }}:${{ env.PLATFORM }},mode=max
cache-from : type=gha,scope=${{ github.workflow }}
cache-to : type=gha,scope=${{ github.workflow }},mode=max
-
name: Import Luxembourg
if : ${{ matrix.mode == 'build-and-test' }}
run : |
docker volume create ${VOLUME}
docker run --rm --shm-size=128M -v ${VOLUME}:${MOUNT} -e UPDATES=enabled ${IMAGE}:${TAG} import
-
name: Start server
if : ${{ matrix.mode == 'build-and-test' }}
run : |
docker run --shm-size=128M -v ${VOLUME}:${MOUNT} -e UPDATES=enabled -p 80:80 -d --name ${CONTAINER} ${IMAGE}:${TAG} run
docker run --rm --shm-size=128M -v ${VOLUME}:${MOUNT} -e UPDATES=enabled -p 80:80 -d --name ${CONTAINER} ${IMAGE}:${TAG} run
sleep 30
docker logs ${CONTAINER}
-
name: Download tiles
if : ${{ matrix.mode == 'build-and-test' }}
run : |
curl http://localhost/tile/0/0/0.png --fail -o 000.png
curl http://localhost/tile/1/0/0.png --fail -o 100.png
@ -90,14 +74,12 @@ jobs:
curl http://localhost/tile/18/135536/89345.png --fail -o example.png
-
name: Upload tiles
if : ${{ matrix.mode == 'build-and-test' }}
uses: actions/upload-artifact@v3
with:
name: tiles
path: '*.png'
-
name: Verify tiles
if : ${{ matrix.mode == 'build-and-test' }}
run : |
sha1sum *.png
sha1sum --check <<EOF
@ -118,7 +100,6 @@ jobs:
done
-
name: Cleanup
if : ${{ matrix.mode == 'build-and-test' }}
run : |
docker rm --force --volumes ${CONTAINER}
docker volume rm --force ${VOLUME}
@ -127,7 +108,7 @@ jobs:
deploy:
runs-on: ubuntu-latest
needs:
- build
- test
if: ${{ github.event_name != 'pull_request' }}
steps:
-
@ -141,7 +122,7 @@ jobs:
-
name: Docker meta
id: meta
uses: docker/metadata-action@v4
uses: docker/metadata-action@v3
with:
images: |
${{ env.DOCKERHUB_IMAGE }}
@ -153,37 +134,33 @@ jobs:
type=semver,pattern={{major}}
-
name: Set up QEMU
uses: docker/setup-qemu-action@v2
with:
platforms: amd64,arm64
uses: docker/setup-qemu-action@v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v1
-
name: Login to DockerHub
uses: docker/login-action@v2
uses: docker/login-action@v1
if: ${{ env.DOCKERHUB_IMAGE != '' }}
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
-
name: Login to GHCR
uses: docker/login-action@v2
uses: docker/login-action@v1
with:
registry : ghcr.io
username : ${{ github.repository_owner }}
password : ${{ secrets.GITHUB_TOKEN }}
-
name: Build and push
uses: docker/build-push-action@v3
uses: docker/build-push-action@v2
with:
pull : true
push : true
platforms : linux/amd64,linux/arm64/v8
context : .
file : ./Dockerfile
tags : ${{ steps.meta.outputs.tags }}
labels : ${{ steps.meta.outputs.labels }}
cache-from: |
type=gha,scope=${{ github.workflow }}:linux/amd64
type=gha,scope=${{ github.workflow }}:linux/arm64/v8
cache-from : type=gha,scope=${{ github.workflow }}
cache-to : type=gha,scope=${{ github.workflow }},mode=max

View File

@ -1,17 +1,11 @@
FROM ubuntu:22.04 AS compiler-common
ENV DEBIAN_FRONTEND=noninteractive
ENV LANG C.UTF-8
ENV LC_ALL C.UTF-8
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
ca-certificates gnupg lsb-release locales \
wget curl \
git-core unzip unrar \
&& locale-gen $LANG && update-locale LANG=$LANG \
&& sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' \
&& wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
&& apt-get update && apt-get -y upgrade
git-core \
ca-certificates \
&& apt-get update
###########################################################################################################
@ -19,10 +13,6 @@ FROM compiler-common AS compiler-stylesheet
RUN cd ~ \
&& git clone --single-branch --branch v5.4.0 https://github.com/gravitystorm/openstreetmap-carto.git --depth 1 \
&& cd openstreetmap-carto \
&& sed -i 's/, "unifont Medium", "Unifont Upper Medium"//g' style/fonts.mss \
&& sed -i 's/"Noto Sans Tibetan Regular",//g' style/fonts.mss \
&& sed -i 's/"Noto Sans Tibetan Bold",//g' style/fonts.mss \
&& sed -i 's/Noto Sans Syriac Eastern Regular/Noto Sans Syriac Regular/g' style/fonts.mss \
&& rm -rf .git
###########################################################################################################
@ -37,17 +27,13 @@ RUN mkdir -p /home/renderer/src \
###########################################################################################################
FROM compiler-common AS final
FROM ubuntu:22.04 AS final
# Based on
# https://switch2osm.org/serving-tiles/manually-building-a-tile-server-18-04-lts/
ENV DEBIAN_FRONTEND=noninteractive
ENV AUTOVACUUM=on
ENV UPDATES=disabled
ENV REPLICATION_URL=https://planet.openstreetmap.org/replication/hour/
ENV MAX_INTERVAL_SECONDS=3600
ENV PG_VERSION 15
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
# Get packages
@ -56,7 +42,6 @@ RUN apt-get update \
apache2 \
cron \
dateutils \
fonts-hanazono \
fonts-noto-cjk \
fonts-noto-hinted \
fonts-noto-unhinted \
@ -70,9 +55,9 @@ RUN apt-get update \
osm2pgsql \
osmium-tool \
osmosis \
postgresql-$PG_VERSION \
postgresql-$PG_VERSION-postgis-3 \
postgresql-$PG_VERSION-postgis-3-scripts \
postgresql-14 \
postgresql-14-postgis-3 \
postgresql-14-postgis-3-scripts \
postgis \
python-is-python3 \
python3-mapnik \
@ -82,19 +67,13 @@ RUN apt-get update \
python3-pip \
renderd \
sudo \
vim \
wget \
&& apt-get clean autoclean \
&& apt-get autoremove --yes \
&& rm -rf /var/lib/{apt,dpkg,cache,log}/
RUN adduser --disabled-password --gecos "" renderer
# Get Noto Emoji Regular font, despite it being deprecated by Google
RUN wget https://github.com/googlefonts/noto-emoji/blob/9a5261d871451f9b5183c93483cbd68ed916b1e9/fonts/NotoEmoji-Regular.ttf?raw=true --content-disposition -P /usr/share/fonts/
# For some reason this one is missing in the default packages
RUN wget https://github.com/stamen/terrain-classic/blob/master/fonts/unifont-Medium.ttf?raw=true --content-disposition -P /usr/share/fonts/
# Install python libraries
RUN pip3 install \
requests \
@ -102,26 +81,17 @@ RUN pip3 install \
pyyaml
# Install carto for stylesheet
RUN npm install -g carto@1.2.0
RUN npm install -g carto@0.18.2
# Configure Apache
RUN echo "LoadModule tile_module /usr/lib/apache2/modules/mod_tile.so" >> /etc/apache2/conf-available/mod_tile.conf \
&& echo "LoadModule headers_module /usr/lib/apache2/modules/mod_headers.so" >> /etc/apache2/conf-available/mod_headers.conf \
&& a2enconf mod_tile && a2enconf mod_headers
COPY apache.conf /etc/apache2/sites-available/000-default.conf
COPY leaflet-demo.html /var/www/html/index.html
RUN ln -sf /dev/stdout /var/log/apache2/access.log \
&& ln -sf /dev/stderr /var/log/apache2/error.log
# leaflet
COPY leaflet-demo.html /var/www/html/index.html
RUN cd /var/www/html/ \
&& wget https://github.com/Leaflet/Leaflet/releases/download/v1.8.0/leaflet.zip \
&& unzip leaflet.zip \
&& rm leaflet.zip
# Icon
RUN wget -O /var/www/html/favicon.ico https://www.openstreetmap.org/favicon.ico
# Copy update scripts
COPY openstreetmap-tiles-update-expire.sh /usr/bin/
RUN chmod +x /usr/bin/openstreetmap-tiles-update-expire.sh \
@ -131,11 +101,11 @@ RUN chmod +x /usr/bin/openstreetmap-tiles-update-expire.sh \
&& echo "* * * * * renderer openstreetmap-tiles-update-expire.sh\n" >> /etc/crontab
# Configure PosgtreSQL
COPY postgresql.custom.conf.tmpl /etc/postgresql/$PG_VERSION/main/
COPY postgresql.custom.conf.tmpl /etc/postgresql/14/main/
RUN chown -R postgres:postgres /var/lib/postgresql \
&& chown postgres:postgres /etc/postgresql/$PG_VERSION/main/postgresql.custom.conf.tmpl \
&& echo "host all all 0.0.0.0/0 scram-sha-256" >> /etc/postgresql/$PG_VERSION/main/pg_hba.conf \
&& echo "host all all ::/0 scram-sha-256" >> /etc/postgresql/$PG_VERSION/main/pg_hba.conf
&& chown postgres:postgres /etc/postgresql/14/main/postgresql.custom.conf.tmpl \
&& echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/14/main/pg_hba.conf \
&& echo "host all all ::/0 md5" >> /etc/postgresql/14/main/pg_hba.conf
# Create volume directories
RUN mkdir -p /run/renderd/ \
@ -145,10 +115,10 @@ RUN mkdir -p /run/renderd/ \
&& chown -R renderer: /data/ \
&& chown -R renderer: /home/renderer/src/ \
&& chown -R renderer: /run/renderd \
&& mv /var/lib/postgresql/$PG_VERSION/main/ /data/database/postgres/ \
&& mv /var/lib/postgresql/14/main/ /data/database/postgres/ \
&& mv /var/cache/renderd/tiles/ /data/tiles/ \
&& chown -R renderer: /data/tiles \
&& ln -s /data/database/postgres /var/lib/postgresql/$PG_VERSION/main \
&& ln -s /data/database/postgres /var/lib/postgresql/14/main \
&& ln -s /data/style /home/renderer/src/openstreetmap-carto \
&& ln -s /data/tiles /var/cache/renderd/tiles \
;
@ -159,8 +129,7 @@ TILEDIR=/var/cache/renderd/tiles \n\
XML=/home/renderer/src/openstreetmap-carto/mapnik.xml \n\
HOST=localhost \n\
TILESIZE=256 \n\
MAXZOOM=20' >> /etc/renderd.conf \
&& sed -i 's,/usr/share/fonts/truetype,/usr/share/fonts,g' /etc/renderd.conf
MAXZOOM=20' >> /etc/renderd.conf
# Install helper script
COPY --from=compiler-helper-script /home/renderer/src/regional /home/renderer/src/regional

View File

@ -67,10 +67,10 @@ By default the container will use openstreetmap-carto if it is not specified. Ho
docker run \
-e DOWNLOAD_PBF=https://download.geofabrik.de/europe/luxembourg-latest.osm.pbf \
-e DOWNLOAD_POLY=https://download.geofabrik.de/europe/luxembourg.poly \
-e NAME_LUA=sample.lua \
-e NAME_STYLE=test.style \
-e NAME_MML=project.mml \
-e NAME_SQL=test.sql \
-e NAME_LUA=sample.lua
-e NAME_STYLE=test.style
-e NAME_MML=project.mml
-e NAME_SQL=test.sql
-v /home/user/openstreetmap-carto-modified:/data/style/ \
-v osm-data:/data/database/ \
overv/openstreetmap-tile-server \
@ -137,29 +137,6 @@ docker run \
This will enable a background process that automatically downloads changes from the OpenStreetMap server, filters them for the relevant region polygon you specified, updates the database and finally marks the affected tiles for rerendering.
### Tile expiration (optional)
Specify custom tile expiration settings to control which zoom level tiles are marked as expired when an update is performed. Tiles can be marked as expired in the cache (TOUCHFROM), but will still be served
until a new tile has been rendered, or deleted from the cache (DELETEFROM), so nothing will be served until a new tile has been rendered.
The example tile expiration values below are the default values.
```
docker run \
-p 8080:80 \
-e REPLICATION_URL=https://planet.openstreetmap.org/replication/minute/ \
-e MAX_INTERVAL_SECONDS=60 \
-e UPDATES=enabled \
-e EXPIRY_MINZOOM=13 \
-e EXPIRY_TOUCHFROM=13 \
-e EXPIRY_DELETEFROM=19 \
-e EXPIRY_MAXZOOM=20 \
-v osm-data:/data/database/ \
-v osm-tiles:/data/tiles/ \
-d overv/openstreetmap-tile-server \
run
```
### Cross-origin resource sharing
To enable the `Access-Control-Allow-Origin` header to be able to retrieve tiles from other domains, simply set the `ALLOW_CORS` variable to `enabled`:

View File

@ -1,25 +0,0 @@
services:
map:
image: git.whimsoc.dev/hendrik/openstreetmap-tile-server:latest
environment:
- UPDATES=enabled
- MAX_INTERVAL_SECONDS=3600
- REPLICATION_URL=https://planet.openstreetmap.org/replication/hour/
- THREADS=10
- OSM2PGSQL_EXTRA_ARGS=-C 8196
- ALLOW_CORS=enabled
- TILESIZE=512
volumes:
- osm-data:/data/database/
- osm-tiles:/data/tiles/
- ./run.sh:/run.sh
ports:
- "8080:80"
command: "run"
shm_size: 2g
volumes:
osm-tiles:
external: true
osm-data:
external: true

14
docker-compose.yml Normal file
View File

@ -0,0 +1,14 @@
version: '3'
services:
map:
image: overv/openstreetmap-tile-server
volumes:
- osm-data:/data/database/
ports:
- "8080:80"
command: "run"
volumes:
osm-data:
external: true

View File

@ -6,8 +6,8 @@
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel="stylesheet" href="leaflet.css"/>
<script src="leaflet.js"></script>
<link rel="stylesheet" href="https://unpkg.com/leaflet@1.7.1/dist/leaflet.css" integrity="sha512-xodZBNTC5n17Xt2atTPuE1HxjVMSvLVW9ocqUKLsCC5CXdbqCmblAshOMAS6/keqq/sMZMZ19scR4PsZChSR7A==" crossorigin="" />
<script src="https://unpkg.com/leaflet@1.7.1/dist/leaflet.js" integrity="sha512-XQoYMqMTK8LvdxXYG3nZ448hOEQiglfqkJs1NOQV44cWnUrBc8PkAOcXy20w0vlaXaVUearIOBhiXZ5V3ynxwA==" crossorigin=""></script>
<style>
html, body, #map {

View File

@ -56,10 +56,14 @@ RUNLOG=$LOG_DIR/run.log
# Here we expire (but don't immediately rerender) tiles between zoom levels
# 13 and 18 and delete between 19 and 20.
#------------------------------------------------------------------------------
EXPIRY_MINZOOM=${EXPIRY_MINZOOM:="13"}
EXPIRY_TOUCHFROM=${EXPIRY_TOUCHFROM:="13"}
EXPIRY_DELETEFROM=${EXPIRY_DELETEFROM:="19"}
EXPIRY_MAXZOOM=${EXPIRY_MAXZOOM:="20"}
EXPIRY_MINZOOM=13
EXPIRY_TOUCHFROM=13
EXPIRY_DELETEFROM=19
EXPIRY_MAXZOOM=20
REPLICATION_URL=${REPLICATION_URL:="https://planet.openstreetmap.org/replication/hour/"}
MAX_INTERVAL_SECONDS=${MAX_INTERVAL_SECONDS:="3600"}
#*************************************************************************
#*************************************************************************
@ -191,7 +195,7 @@ m_ok "expiring tiles"
# delete >= $EXPIRY_DELETEFROM and <= $EXPIRY_MAXZOOM.
# The default path to renderd.sock is fixed.
#------------------------------------------------------------------------------
if ! render_expired --map=default --min-zoom=$EXPIRY_MINZOOM --touch-from=$EXPIRY_TOUCHFROM --delete-from=$EXPIRY_DELETEFROM --max-zoom=$EXPIRY_MAXZOOM -s /run/renderd/renderd.sock < "$EXPIRY_FILE.$$" 2>&1 | tail -8 >> "$EXPIRYLOG"; then
if ! render_expired --map=default --min-zoom=$EXPIRY_MINZOOM --touch-from=$EXPIRY_TOUCHFROM --delete-from=$EXPIRY_DELETEFROM --max-zoom=$EXPIRY_MAXZOOM -s /var/run/renderd/renderd.sock < "$EXPIRY_FILE.$$" 2>&1 | tail -8 >> "$EXPIRYLOG"; then
m_info "Expiry failed"
fi

View File

@ -1,7 +1,7 @@
# Suggested minimal settings from
# https://ircama.github.io/osm-carto-tutorials/tile-server-ubuntu/
shared_buffers = 1GB
shared_buffers = 128MB
min_wal_size = 1GB
# max_wal_size = 2GB # Overridden below
maintenance_work_mem = 256MB
@ -16,14 +16,10 @@ wal_buffers = 1024kB
wal_writer_delay = 500ms
commit_delay = 10000
# checkpoint_segments = 60 # unrecognized in psql 10.7.1
max_wal_size = 5GB
max_wal_size = 2880MB
random_page_cost = 1.1
track_activity_query_size = 16384
autovacuum_vacuum_scale_factor = 0.05
autovacuum_analyze_scale_factor = 0.02
checkpoint_timeout = 60min
wal_level = minimal
max_wal_senders = 0
listen_addresses = '*'

15
run.sh
View File

@ -3,9 +3,9 @@
set -euo pipefail
function createPostgresConfig() {
cp /etc/postgresql/$PG_VERSION/main/postgresql.custom.conf.tmpl /etc/postgresql/$PG_VERSION/main/conf.d/postgresql.custom.conf
sudo -u postgres echo "autovacuum = $AUTOVACUUM" >> /etc/postgresql/$PG_VERSION/main/conf.d/postgresql.custom.conf
cat /etc/postgresql/$PG_VERSION/main/conf.d/postgresql.custom.conf
cp /etc/postgresql/14/main/postgresql.custom.conf.tmpl /etc/postgresql/14/main/conf.d/postgresql.custom.conf
sudo -u postgres echo "autovacuum = $AUTOVACUUM" >> /etc/postgresql/14/main/conf.d/postgresql.custom.conf
cat /etc/postgresql/14/main/conf.d/postgresql.custom.conf
}
function setPostgresPassword() {
@ -46,7 +46,7 @@ if [ "$1" == "import" ]; then
chown renderer: /data/database/
chown -R postgres: /var/lib/postgresql /data/database/postgres/
if [ ! -f /data/database/postgres/PG_VERSION ]; then
sudo -u postgres /usr/lib/postgresql/$PG_VERSION/bin/pg_ctl -D /data/database/postgres/ initdb -o "--locale C.UTF-8"
sudo -u postgres /usr/lib/postgresql/14/bin/pg_ctl -D /data/database/postgres/ initdb -o "--locale C.UTF-8"
fi
# Initialize PostgreSQL
@ -78,10 +78,10 @@ if [ "$1" == "import" ]; then
if [ "${UPDATES:-}" == "enabled" ] || [ "${UPDATES:-}" == "1" ]; then
# determine and set osmosis_replication_timestamp (for consecutive updates)
REPLICATION_TIMESTAMP=`osmium fileinfo -g header.option.osmosis_replication_timestamp /data/region.osm.pbf`
REPLICATION_TIMESTAMP=`osmium fileinfo /data/region.osm.pbf | grep 'osmosis_replication_timestamp=' | cut -d "=" -f 2`
# initial setup of osmosis workspace (for consecutive updates)
sudo -E -u renderer openstreetmap-tiles-update-expire.sh $REPLICATION_TIMESTAMP
sudo -u renderer openstreetmap-tiles-update-expire.sh $REPLICATION_TIMESTAMP
fi
# copy polygon file if available
@ -169,7 +169,6 @@ if [ "$1" == "run" ]; then
# Configure renderd threads
sed -i -E "s/num_threads=[0-9]+/num_threads=${THREADS:-4}/g" /etc/renderd.conf
sed -i -E "s/TILESIZE=[0-9]+/TILESIZE=${TILESIZE:-256}/g" /etc/renderd.conf
# start cron job to trigger consecutive updates
if [ "${UPDATES:-}" == "enabled" ] || [ "${UPDATES:-}" == "1" ]; then
@ -198,5 +197,3 @@ fi
echo "invalid command"
exit 1