From 50e60e634d70843ae5f5eaaa1a2da24096a4c032 Mon Sep 17 00:00:00 2001 From: souladm Date: Mon, 27 May 2019 14:36:14 +0300 Subject: [PATCH] Added postgresql indexes --- Dockerfile | 76 ++++++++++++++---------------------------------------- README.md | 16 +++++++----- run.sh | 44 +++++++++++++++++++++++++------ 3 files changed, 65 insertions(+), 71 deletions(-) diff --git a/Dockerfile b/Dockerfile index eb7f98e..8f4e03f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,9 +2,14 @@ FROM ubuntu:18.04 # Based on # https://switch2osm.org/manually-building-a-tile-server-18-04-lts/ - +LABEL maintainer="kosar@freedom.valor.ua" +LABEL baseImage="overv/openstreetmap-tile-server" +LABEL vendor="1node" +LABEL created="27-05-2019" +LABEL lastUpdate="27-05-2019" +LABEL description="OSM-tile-server container based on UBUNTU 18.04" # Set up environment -ENV TZ=UTC +ENV TZ=Europe/Kiev ENV AUTOVACUUM=on RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone @@ -12,60 +17,19 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone RUN echo "deb [ allow-insecure=yes ] http://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main" >> /etc/apt/sources.list.d/pgdg.list \ && apt-get update \ && apt-get install -y apt-transport-https ca-certificates \ - && apt-get install -y --no-install-recommends --allow-unauthenticated \ - apache2 \ - apache2-dev \ - autoconf \ - build-essential \ - bzip2 \ - cmake \ - fonts-noto-cjk \ - fonts-noto-hinted \ - fonts-noto-unhinted \ - clang \ - gdal-bin \ - git-core \ - libagg-dev \ - libboost-all-dev \ - libbz2-dev \ - libcairo-dev \ - libcairomm-1.0-dev \ - libexpat1-dev \ - libfreetype6-dev \ - libgdal-dev \ - libgeos++-dev \ - libgeos-dev \ - libgeotiff-epsg \ - libicu-dev \ - liblua5.3-dev \ - libmapnik-dev \ - libpq-dev \ - libproj-dev \ - libprotobuf-c0-dev \ - libtiff5-dev \ - libtool \ - libxml2-dev \ - lua5.3 \ - make \ - mapnik-utils \ - nodejs \ - npm \ - postgis \ - postgresql-10 \ - postgresql-10-postgis-2.5 \ - postgresql-10-postgis-2.5-scripts \ - postgresql-contrib-10 \ - protobuf-c-compiler \ - python-mapnik \ - sudo \ - tar \ - ttf-unifont \ - unzip \ - wget \ - zlib1g-dev \ -&& apt-get clean autoclean \ -&& apt-get autoremove --yes \ -&& rm -rf /var/lib/{apt,dpkg,cache,log}/ + && apt-get install -y --no-install-recommends --allow-unauthenticated apache2 \ + apache2-dev autoconf build-essential bzip2 cmake fonts-noto-cjk \ + fonts-noto-hinted fonts-noto-unhinted clang gdal-bin git-core libagg-dev \ + libboost-all-dev libbz2-dev libcairo-dev libcairomm-1.0-dev libexpat1-dev \ + libfreetype6-dev libgdal-dev libgeos++-dev libgeos-dev libgeotiff-epsg \ + libicu-dev liblua5.3-dev libmapnik-dev libpq-dev libproj-dev libprotobuf-c0-dev \ + libtiff5-dev libtool libxml2-dev lua5.3 make mapnik-utils nodejs npm \ + postgis postgresql-10 postgresql-10-postgis-2.5 postgresql-10-postgis-2.5-scripts \ + postgresql-contrib-10 protobuf-c-compiler python-mapnik sudo tar \ + ttf-unifont unzip wget zlib1g-dev \ + && apt-get clean autoclean \ + && apt-get autoremove --yes \ + && rm -rf /var/lib/{apt,dpkg,cache,log} # Set up renderer user RUN adduser --disabled-password --gecos "" renderer diff --git a/README.md b/README.md index ea3ddd0..15cad9b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ # openstreetmap-tile-server +BASED ON https://github.com/Overv/openstreetmap-tile-server +ADDED: postgresql indexes to make it faster. This container allows you to easily set up an OpenStreetMap PNG tile server given a `.osm.pbf` file. It is based on the [latest Ubuntu 18.04 LTS guide](https://switch2osm.org/manually-building-a-tile-server-18-04-lts/) from [switch2osm.org](https://switch2osm.org/) and therefore uses the default OpenStreetMap style. @@ -10,7 +12,7 @@ First create a Docker volume to hold the PostgreSQL database that will contain t Next, download an .osm.pbf extract from geofabrik.de for the region that you're interested in. You can then start importing it into PostgreSQL by running a container and mounting the file as `/data.osm.pbf`. For example: - docker run -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf -v openstreetmap-data:/var/lib/postgresql/10/main overv/openstreetmap-tile-server import + docker run -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf -v openstreetmap-data:/var/lib/postgresql/10/main kosar/openstreetmap-tile-server import If the container exits without errors, then your data has been successfully imported and you are now ready to run the tile server. @@ -18,7 +20,7 @@ If the container exits without errors, then your data has been successfully impo Run the server like this: - docker run -p 80:80 -v openstreetmap-data:/var/lib/postgresql/10/main -d overv/openstreetmap-tile-server run + docker run -p 80:80 -v openstreetmap-data:/var/lib/postgresql/10/main -d kosar/openstreetmap-tile-server run Your tiles will now be available at `http://localhost:80/tile/{z}/{x}/{y}.png`. The demo map in `leaflet-demo.html` will then be available on `http://localhost:80`. Note that it will initially quite a bit of time to render the larger tiles for the first time. @@ -27,7 +29,7 @@ Your tiles will now be available at `http://localhost:80/tile/{z}/{x}/{y}.png`. Tiles that have already been rendered will be stored in `/var/lib/mod_tile`. To make sure that this data survives container restarts, you should create another volume for it: docker volume create openstreetmap-rendered-tiles - docker run -p 80:80 -v openstreetmap-data:/var/lib/postgresql/10/main -v openstreetmap-rendered-tiles:/var/lib/mod_tile -d overv/openstreetmap-tile-server run + docker run -p 80:80 -v openstreetmap-data:/var/lib/postgresql/10/main -v openstreetmap-rendered-tiles:/var/lib/mod_tile -d kosar/openstreetmap-tile-server run ## Performance tuning @@ -35,13 +37,13 @@ Tiles that have already been rendered will be stored in `/var/lib/mod_tile`. To The import and tile serving processes use 4 threads by default, but this number can be changed by setting the `THREADS` environment variable. For example: - docker run -p 80:80 -e THREADS=24 -v openstreetmap-data:/var/lib/postgresql/10/main -d overv/openstreetmap-tile-server run + docker run -p 80:80 -e THREADS=24 -v openstreetmap-data:/var/lib/postgresql/10/main -d kosar/openstreetmap-tile-server run ### AUTOVACUUM The database use the autovacuum feature by default. This behavior can be changed with `AUTOVACUUM` environment variable. For example: - docker run -p 80:80 -e AUTOVACUUM=off -v openstreetmap-data:/var/lib/postgresql/10/main -d overv/openstreetmap-tile-server + docker run -p 80:80 -e AUTOVACUUM=off -v openstreetmap-data:/var/lib/postgresql/10/main -d kosar/openstreetmap-tile-server ## Benchmarks @@ -58,14 +60,14 @@ If you encounter such entries in the log, it will mean that the default shared m To raise it use `--shm-size` parameter. For example: - docker run -p 80:80 -v openstreetmap-data:/var/lib/postgresql/10/main --shm-size="192m" -d overv/openstreetmap-tile-server run + docker run -p 80:80 -v openstreetmap-data:/var/lib/postgresql/10/main --shm-size="192m" -d kosar/openstreetmap-tile-server run For too high values you may notice excessive CPU load and memory usage. It might be that you will have to experimentally find the best values for yourself. ## License ``` -Copyright 2019 Alexander Overvoorde +Copyright 2019 Bogdan Kosarevskyi Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/run.sh b/run.sh index d7c0d01..06b5998 100755 --- a/run.sh +++ b/run.sh @@ -23,17 +23,45 @@ if [ "$1" = "import" ]; then # Initialize PostgreSQL CreatePostgressqlConfig service postgresql start - sudo -u postgres createuser renderer - sudo -u postgres createdb -E UTF8 -O renderer gis - sudo -u postgres psql -d gis -c "CREATE EXTENSION postgis;" - sudo -u postgres psql -d gis -c "CREATE EXTENSION hstore;" - sudo -u postgres psql -d gis -c "ALTER TABLE geometry_columns OWNER TO renderer;" - sudo -u postgres psql -d gis -c "ALTER TABLE spatial_ref_sys OWNER TO renderer;" - + su postgres << EOSU + createuser renderer + createdb -E UTF8 -O renderer gis + psql -d gis -c "CREATE EXTENSION postgis;" + psql -d gis -c "CREATE EXTENSION hstore;" + psql -d gis -c "ALTER TABLE geometry_columns OWNER TO renderer;" + psql -d gis -c "ALTER TABLE spatial_ref_sys OWNER TO renderer;" + psql -d gis -c "CREATE INDEX planet_osm_roads_admin ON planet_osm_roads USING GIST (way) + WHERE boundary = 'administrative';" + psql -d gis -c "CREATE INDEX planet_osm_roads_roads_ref ON planet_osm_roads USING GIST (way) + WHERE highway IS NOT NULL AND ref IS NOT NULL;" + psql -d gis -c "CREATE INDEX planet_osm_roads_admin_low ON planet_osm_roads USING GIST (way) + WHERE boundary = 'administrative' AND admin_level IN ('0', '1', '2', '3', '4');" + psql -d gis -c "CREATE INDEX planet_osm_line_ferry ON planet_osm_line USING GIST (way) + WHERE route = 'ferry';" + psql -d gis -c "CREATE INDEX planet_osm_line_river ON planet_osm_line USING GIST (way) + WHERE waterway = 'river';" + psql -d gis -c "CREATE INDEX planet_osm_line_name ON planet_osm_line USING GIST (way) + WHERE name IS NOT NULL;" + psql -d gis -c "CREATE INDEX planet_osm_polygon_water ON planet_osm_polygon USING GIST (way) + WHERE waterway IN ('dock', 'riverbank', 'canal') OR landuse IN ('reservoir', 'basin') + OR \"natural\" IN ('water', 'glacier');" + psql -d gis -c "CREATE INDEX planet_osm_polygon_nobuilding ON planet_osm_polygon USING GIST (way) + WHERE building IS NULL;" + psql -d gis -c "CREATE INDEX planet_osm_polygon_name ON planet_osm_polygon USING GIST (way) + WHERE name IS NOT NULL;" + psql -d gis -c "CREATE INDEX planet_osm_polygon_way_area_z10 ON planet_osm_polygon USING GIST (way) + WHERE way_area > 23300;" + psql -d gis -c "CREATE INDEX planet_osm_polygon_military ON planet_osm_polygon USING GIST (way) + WHERE (landuse = 'military' OR military = 'danger_area') AND building IS NULL;" + psql -d gis -c "CREATE INDEX planet_osm_polygon_way_area_z6 ON planet_osm_polygon USING GIST (way) + WHERE way_area > 5980000;" + psql -d gis -c "CREATE INDEX planet_osm_point_place ON planet_osm_point USING GIST (way) + WHERE place IS NOT NULL AND name IS NOT NULL;" +EOSU # Download Luxembourg as sample if no data is provided if [ ! -f /data.osm.pbf ]; then echo "WARNING: No import file at /data.osm.pbf, so importing Luxembourg as example..." - wget -nv http://download.geofabrik.de/europe/luxembourg-latest.osm.pbf -O /data.osm.pbf + wget -nv https://download.geofabrik.de/europe/ukraine-latest.osm.pbf -O /data.osm.pbf fi # Import data