Added postgresql indexes

This commit is contained in:
souladm 2019-05-27 14:36:14 +03:00
parent d7c2817f9f
commit 50e60e634d
3 changed files with 65 additions and 71 deletions

View File

@ -2,9 +2,14 @@ FROM ubuntu:18.04
# Based on # Based on
# https://switch2osm.org/manually-building-a-tile-server-18-04-lts/ # https://switch2osm.org/manually-building-a-tile-server-18-04-lts/
LABEL maintainer="kosar@freedom.valor.ua"
LABEL baseImage="overv/openstreetmap-tile-server"
LABEL vendor="1node"
LABEL created="27-05-2019"
LABEL lastUpdate="27-05-2019"
LABEL description="OSM-tile-server container based on UBUNTU 18.04"
# Set up environment # Set up environment
ENV TZ=UTC ENV TZ=Europe/Kiev
ENV AUTOVACUUM=on ENV AUTOVACUUM=on
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
@ -12,60 +17,19 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
RUN echo "deb [ allow-insecure=yes ] http://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main" >> /etc/apt/sources.list.d/pgdg.list \ RUN echo "deb [ allow-insecure=yes ] http://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main" >> /etc/apt/sources.list.d/pgdg.list \
&& apt-get update \ && apt-get update \
&& apt-get install -y apt-transport-https ca-certificates \ && apt-get install -y apt-transport-https ca-certificates \
&& apt-get install -y --no-install-recommends --allow-unauthenticated \ && apt-get install -y --no-install-recommends --allow-unauthenticated apache2 \
apache2 \ apache2-dev autoconf build-essential bzip2 cmake fonts-noto-cjk \
apache2-dev \ fonts-noto-hinted fonts-noto-unhinted clang gdal-bin git-core libagg-dev \
autoconf \ libboost-all-dev libbz2-dev libcairo-dev libcairomm-1.0-dev libexpat1-dev \
build-essential \ libfreetype6-dev libgdal-dev libgeos++-dev libgeos-dev libgeotiff-epsg \
bzip2 \ libicu-dev liblua5.3-dev libmapnik-dev libpq-dev libproj-dev libprotobuf-c0-dev \
cmake \ libtiff5-dev libtool libxml2-dev lua5.3 make mapnik-utils nodejs npm \
fonts-noto-cjk \ postgis postgresql-10 postgresql-10-postgis-2.5 postgresql-10-postgis-2.5-scripts \
fonts-noto-hinted \ postgresql-contrib-10 protobuf-c-compiler python-mapnik sudo tar \
fonts-noto-unhinted \ ttf-unifont unzip wget zlib1g-dev \
clang \ && apt-get clean autoclean \
gdal-bin \ && apt-get autoremove --yes \
git-core \ && rm -rf /var/lib/{apt,dpkg,cache,log}
libagg-dev \
libboost-all-dev \
libbz2-dev \
libcairo-dev \
libcairomm-1.0-dev \
libexpat1-dev \
libfreetype6-dev \
libgdal-dev \
libgeos++-dev \
libgeos-dev \
libgeotiff-epsg \
libicu-dev \
liblua5.3-dev \
libmapnik-dev \
libpq-dev \
libproj-dev \
libprotobuf-c0-dev \
libtiff5-dev \
libtool \
libxml2-dev \
lua5.3 \
make \
mapnik-utils \
nodejs \
npm \
postgis \
postgresql-10 \
postgresql-10-postgis-2.5 \
postgresql-10-postgis-2.5-scripts \
postgresql-contrib-10 \
protobuf-c-compiler \
python-mapnik \
sudo \
tar \
ttf-unifont \
unzip \
wget \
zlib1g-dev \
&& apt-get clean autoclean \
&& apt-get autoremove --yes \
&& rm -rf /var/lib/{apt,dpkg,cache,log}/
# Set up renderer user # Set up renderer user
RUN adduser --disabled-password --gecos "" renderer RUN adduser --disabled-password --gecos "" renderer

View File

@ -1,4 +1,6 @@
# openstreetmap-tile-server # openstreetmap-tile-server
BASED ON https://github.com/Overv/openstreetmap-tile-server
ADDED: postgresql indexes to make it faster.
This container allows you to easily set up an OpenStreetMap PNG tile server given a `.osm.pbf` file. It is based on the [latest Ubuntu 18.04 LTS guide](https://switch2osm.org/manually-building-a-tile-server-18-04-lts/) from [switch2osm.org](https://switch2osm.org/) and therefore uses the default OpenStreetMap style. This container allows you to easily set up an OpenStreetMap PNG tile server given a `.osm.pbf` file. It is based on the [latest Ubuntu 18.04 LTS guide](https://switch2osm.org/manually-building-a-tile-server-18-04-lts/) from [switch2osm.org](https://switch2osm.org/) and therefore uses the default OpenStreetMap style.
@ -10,7 +12,7 @@ First create a Docker volume to hold the PostgreSQL database that will contain t
Next, download an .osm.pbf extract from geofabrik.de for the region that you're interested in. You can then start importing it into PostgreSQL by running a container and mounting the file as `/data.osm.pbf`. For example: Next, download an .osm.pbf extract from geofabrik.de for the region that you're interested in. You can then start importing it into PostgreSQL by running a container and mounting the file as `/data.osm.pbf`. For example:
docker run -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf -v openstreetmap-data:/var/lib/postgresql/10/main overv/openstreetmap-tile-server import docker run -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf -v openstreetmap-data:/var/lib/postgresql/10/main kosar/openstreetmap-tile-server import
If the container exits without errors, then your data has been successfully imported and you are now ready to run the tile server. If the container exits without errors, then your data has been successfully imported and you are now ready to run the tile server.
@ -18,7 +20,7 @@ If the container exits without errors, then your data has been successfully impo
Run the server like this: Run the server like this:
docker run -p 80:80 -v openstreetmap-data:/var/lib/postgresql/10/main -d overv/openstreetmap-tile-server run docker run -p 80:80 -v openstreetmap-data:/var/lib/postgresql/10/main -d kosar/openstreetmap-tile-server run
Your tiles will now be available at `http://localhost:80/tile/{z}/{x}/{y}.png`. The demo map in `leaflet-demo.html` will then be available on `http://localhost:80`. Note that it will initially quite a bit of time to render the larger tiles for the first time. Your tiles will now be available at `http://localhost:80/tile/{z}/{x}/{y}.png`. The demo map in `leaflet-demo.html` will then be available on `http://localhost:80`. Note that it will initially quite a bit of time to render the larger tiles for the first time.
@ -27,7 +29,7 @@ Your tiles will now be available at `http://localhost:80/tile/{z}/{x}/{y}.png`.
Tiles that have already been rendered will be stored in `/var/lib/mod_tile`. To make sure that this data survives container restarts, you should create another volume for it: Tiles that have already been rendered will be stored in `/var/lib/mod_tile`. To make sure that this data survives container restarts, you should create another volume for it:
docker volume create openstreetmap-rendered-tiles docker volume create openstreetmap-rendered-tiles
docker run -p 80:80 -v openstreetmap-data:/var/lib/postgresql/10/main -v openstreetmap-rendered-tiles:/var/lib/mod_tile -d overv/openstreetmap-tile-server run docker run -p 80:80 -v openstreetmap-data:/var/lib/postgresql/10/main -v openstreetmap-rendered-tiles:/var/lib/mod_tile -d kosar/openstreetmap-tile-server run
## Performance tuning ## Performance tuning
@ -35,13 +37,13 @@ Tiles that have already been rendered will be stored in `/var/lib/mod_tile`. To
The import and tile serving processes use 4 threads by default, but this number can be changed by setting the `THREADS` environment variable. For example: The import and tile serving processes use 4 threads by default, but this number can be changed by setting the `THREADS` environment variable. For example:
docker run -p 80:80 -e THREADS=24 -v openstreetmap-data:/var/lib/postgresql/10/main -d overv/openstreetmap-tile-server run docker run -p 80:80 -e THREADS=24 -v openstreetmap-data:/var/lib/postgresql/10/main -d kosar/openstreetmap-tile-server run
### AUTOVACUUM ### AUTOVACUUM
The database use the autovacuum feature by default. This behavior can be changed with `AUTOVACUUM` environment variable. For example: The database use the autovacuum feature by default. This behavior can be changed with `AUTOVACUUM` environment variable. For example:
docker run -p 80:80 -e AUTOVACUUM=off -v openstreetmap-data:/var/lib/postgresql/10/main -d overv/openstreetmap-tile-server docker run -p 80:80 -e AUTOVACUUM=off -v openstreetmap-data:/var/lib/postgresql/10/main -d kosar/openstreetmap-tile-server
## Benchmarks ## Benchmarks
@ -58,14 +60,14 @@ If you encounter such entries in the log, it will mean that the default shared m
To raise it use `--shm-size` parameter. For example: To raise it use `--shm-size` parameter. For example:
docker run -p 80:80 -v openstreetmap-data:/var/lib/postgresql/10/main --shm-size="192m" -d overv/openstreetmap-tile-server run docker run -p 80:80 -v openstreetmap-data:/var/lib/postgresql/10/main --shm-size="192m" -d kosar/openstreetmap-tile-server run
For too high values you may notice excessive CPU load and memory usage. It might be that you will have to experimentally find the best values for yourself. For too high values you may notice excessive CPU load and memory usage. It might be that you will have to experimentally find the best values for yourself.
## License ## License
``` ```
Copyright 2019 Alexander Overvoorde Copyright 2019 Bogdan Kosarevskyi
Licensed under the Apache License, Version 2.0 (the "License"); Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License. you may not use this file except in compliance with the License.

44
run.sh
View File

@ -23,17 +23,45 @@ if [ "$1" = "import" ]; then
# Initialize PostgreSQL # Initialize PostgreSQL
CreatePostgressqlConfig CreatePostgressqlConfig
service postgresql start service postgresql start
sudo -u postgres createuser renderer su postgres << EOSU
sudo -u postgres createdb -E UTF8 -O renderer gis createuser renderer
sudo -u postgres psql -d gis -c "CREATE EXTENSION postgis;" createdb -E UTF8 -O renderer gis
sudo -u postgres psql -d gis -c "CREATE EXTENSION hstore;" psql -d gis -c "CREATE EXTENSION postgis;"
sudo -u postgres psql -d gis -c "ALTER TABLE geometry_columns OWNER TO renderer;" psql -d gis -c "CREATE EXTENSION hstore;"
sudo -u postgres psql -d gis -c "ALTER TABLE spatial_ref_sys OWNER TO renderer;" psql -d gis -c "ALTER TABLE geometry_columns OWNER TO renderer;"
psql -d gis -c "ALTER TABLE spatial_ref_sys OWNER TO renderer;"
psql -d gis -c "CREATE INDEX planet_osm_roads_admin ON planet_osm_roads USING GIST (way)
WHERE boundary = 'administrative';"
psql -d gis -c "CREATE INDEX planet_osm_roads_roads_ref ON planet_osm_roads USING GIST (way)
WHERE highway IS NOT NULL AND ref IS NOT NULL;"
psql -d gis -c "CREATE INDEX planet_osm_roads_admin_low ON planet_osm_roads USING GIST (way)
WHERE boundary = 'administrative' AND admin_level IN ('0', '1', '2', '3', '4');"
psql -d gis -c "CREATE INDEX planet_osm_line_ferry ON planet_osm_line USING GIST (way)
WHERE route = 'ferry';"
psql -d gis -c "CREATE INDEX planet_osm_line_river ON planet_osm_line USING GIST (way)
WHERE waterway = 'river';"
psql -d gis -c "CREATE INDEX planet_osm_line_name ON planet_osm_line USING GIST (way)
WHERE name IS NOT NULL;"
psql -d gis -c "CREATE INDEX planet_osm_polygon_water ON planet_osm_polygon USING GIST (way)
WHERE waterway IN ('dock', 'riverbank', 'canal') OR landuse IN ('reservoir', 'basin')
OR \"natural\" IN ('water', 'glacier');"
psql -d gis -c "CREATE INDEX planet_osm_polygon_nobuilding ON planet_osm_polygon USING GIST (way)
WHERE building IS NULL;"
psql -d gis -c "CREATE INDEX planet_osm_polygon_name ON planet_osm_polygon USING GIST (way)
WHERE name IS NOT NULL;"
psql -d gis -c "CREATE INDEX planet_osm_polygon_way_area_z10 ON planet_osm_polygon USING GIST (way)
WHERE way_area > 23300;"
psql -d gis -c "CREATE INDEX planet_osm_polygon_military ON planet_osm_polygon USING GIST (way)
WHERE (landuse = 'military' OR military = 'danger_area') AND building IS NULL;"
psql -d gis -c "CREATE INDEX planet_osm_polygon_way_area_z6 ON planet_osm_polygon USING GIST (way)
WHERE way_area > 5980000;"
psql -d gis -c "CREATE INDEX planet_osm_point_place ON planet_osm_point USING GIST (way)
WHERE place IS NOT NULL AND name IS NOT NULL;"
EOSU
# Download Luxembourg as sample if no data is provided # Download Luxembourg as sample if no data is provided
if [ ! -f /data.osm.pbf ]; then if [ ! -f /data.osm.pbf ]; then
echo "WARNING: No import file at /data.osm.pbf, so importing Luxembourg as example..." echo "WARNING: No import file at /data.osm.pbf, so importing Luxembourg as example..."
wget -nv http://download.geofabrik.de/europe/luxembourg-latest.osm.pbf -O /data.osm.pbf wget -nv https://download.geofabrik.de/europe/ukraine-latest.osm.pbf -O /data.osm.pbf
fi fi
# Import data # Import data