diff --git a/Dockerfile b/Dockerfile index 883248f..fb675fb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,36 +4,36 @@ ENV LANG C.UTF-8 ENV LC_ALL C.UTF-8 RUN apt-get update \ -&& apt-get install -y --no-install-recommends \ - ca-certificates gnupg lsb-release locales \ - wget curl \ - git-core unzip unrar \ -&& locale-gen $LANG && update-locale LANG=$LANG \ -&& sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' \ -&& wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ -&& apt-get update && apt-get -y upgrade + && apt-get install -y --no-install-recommends \ + ca-certificates gnupg lsb-release locales \ + wget curl \ + git-core unzip unrar \ + && locale-gen $LANG && update-locale LANG=$LANG \ + && sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' \ + && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ + && apt-get update && apt-get -y upgrade ########################################################################################################### FROM compiler-common AS compiler-stylesheet RUN cd ~ \ -&& git clone --single-branch --branch v5.4.0 https://github.com/gravitystorm/openstreetmap-carto.git --depth 1 \ -&& cd openstreetmap-carto \ -&& sed -i 's/, "unifont Medium", "Unifont Upper Medium"//g' style/fonts.mss \ -&& sed -i 's/"Noto Sans Tibetan Regular",//g' style/fonts.mss \ -&& sed -i 's/"Noto Sans Tibetan Bold",//g' style/fonts.mss \ -&& sed -i 's/Noto Sans Syriac Eastern Regular/Noto Sans Syriac Regular/g' style/fonts.mss \ -&& rm -rf .git + && git clone --single-branch --branch v5.4.0 https://github.com/gravitystorm/openstreetmap-carto.git --depth 1 \ + && cd openstreetmap-carto \ + && sed -i 's/, "unifont Medium", "Unifont Upper Medium"//g' style/fonts.mss \ + && sed -i 's/"Noto Sans Tibetan Regular",//g' style/fonts.mss \ + && sed -i 's/"Noto Sans Tibetan Bold",//g' style/fonts.mss \ + && sed -i 's/Noto Sans Syriac Eastern Regular/Noto Sans Syriac Regular/g' style/fonts.mss \ + && rm -rf .git ########################################################################################################### FROM compiler-common AS compiler-helper-script RUN mkdir -p /home/renderer/src \ -&& cd /home/renderer/src \ -&& git clone https://github.com/zverik/regional \ -&& cd regional \ -&& rm -rf .git \ -&& chmod u+x /home/renderer/src/regional/trim_osc.py + && cd /home/renderer/src \ + && git clone https://github.com/zverik/regional \ + && cd regional \ + && rm -rf .git \ + && chmod u+x /home/renderer/src/regional/trim_osc.py ########################################################################################################### @@ -52,40 +52,41 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone # Get packages RUN apt-get update \ -&& apt-get install -y --no-install-recommends \ - apache2 \ - cron \ - dateutils \ - fonts-hanazono \ - fonts-noto-cjk \ - fonts-noto-hinted \ - fonts-noto-unhinted \ - fonts-unifont \ - gnupg2 \ - gdal-bin \ - liblua5.3-dev \ - lua5.3 \ - mapnik-utils \ - npm \ - osm2pgsql \ - osmium-tool \ - osmosis \ - postgresql-$PG_VERSION \ - postgresql-$PG_VERSION-postgis-3 \ - postgresql-$PG_VERSION-postgis-3-scripts \ - postgis \ - python-is-python3 \ - python3-mapnik \ - python3-lxml \ - python3-psycopg2 \ - python3-shapely \ - python3-pip \ - renderd \ - sudo \ - vim \ -&& apt-get clean autoclean \ -&& apt-get autoremove --yes \ -&& rm -rf /var/lib/{apt,dpkg,cache,log}/ + && apt-get install -y --no-install-recommends \ + apache2 \ + cron \ + dateutils \ + fonts-hanazono \ + fonts-noto-cjk \ + fonts-noto-hinted \ + fonts-noto-unhinted \ + fonts-unifont \ + gnupg2 \ + gdal-bin \ + liblua5.3-dev \ + lua5.3 \ + mapnik-utils \ + npm \ + osm2pgsql \ + osmium-tool \ + osmosis \ + postgresql-$PG_VERSION \ + postgresql-$PG_VERSION-postgis-3 \ + postgresql-$PG_VERSION-postgis-3-scripts \ + postgis \ + python-is-python3 \ + python3-mapnik \ + python3-lxml \ + python3-psycopg2 \ + python3-shapely \ + python3-pip \ + renderd \ + sudo \ + vim \ + openssh-client \ + && apt-get clean autoclean \ + && apt-get autoremove --yes \ + && rm -rf /var/lib/{apt,dpkg,cache,log}/ RUN adduser --disabled-password --gecos "" renderer @@ -97,27 +98,27 @@ RUN wget https://github.com/stamen/terrain-classic/blob/master/fonts/unifont-Med # Install python libraries RUN pip3 install \ - requests \ - osmium \ - pyyaml + requests \ + osmium \ + pyyaml # Install carto for stylesheet RUN npm install -g carto@1.2.0 # Configure Apache RUN echo "LoadModule tile_module /usr/lib/apache2/modules/mod_tile.so" >> /etc/apache2/conf-available/mod_tile.conf \ -&& echo "LoadModule headers_module /usr/lib/apache2/modules/mod_headers.so" >> /etc/apache2/conf-available/mod_headers.conf \ -&& a2enconf mod_tile && a2enconf mod_headers + && echo "LoadModule headers_module /usr/lib/apache2/modules/mod_headers.so" >> /etc/apache2/conf-available/mod_headers.conf \ + && a2enconf mod_tile && a2enconf mod_headers COPY apache.conf /etc/apache2/sites-available/000-default.conf RUN ln -sf /dev/stdout /var/log/apache2/access.log \ -&& ln -sf /dev/stderr /var/log/apache2/error.log + && ln -sf /dev/stderr /var/log/apache2/error.log # leaflet COPY leaflet-demo.html /var/www/html/index.html RUN cd /var/www/html/ \ -&& wget https://github.com/Leaflet/Leaflet/releases/download/v1.8.0/leaflet.zip \ -&& unzip leaflet.zip \ -&& rm leaflet.zip + && wget https://github.com/Leaflet/Leaflet/releases/download/v1.8.0/leaflet.zip \ + && unzip leaflet.zip \ + && rm leaflet.zip # Icon RUN wget -O /var/www/html/favicon.ico https://www.openstreetmap.org/favicon.ico @@ -125,17 +126,17 @@ RUN wget -O /var/www/html/favicon.ico https://www.openstreetmap.org/favicon.ico # Copy update scripts COPY openstreetmap-tiles-update-expire.sh /usr/bin/ RUN chmod +x /usr/bin/openstreetmap-tiles-update-expire.sh \ -&& mkdir /var/log/tiles \ -&& chmod a+rw /var/log/tiles \ -&& ln -s /home/renderer/src/mod_tile/osmosis-db_replag /usr/bin/osmosis-db_replag \ -&& echo "* * * * * renderer openstreetmap-tiles-update-expire.sh\n" >> /etc/crontab + && mkdir /var/log/tiles \ + && chmod a+rw /var/log/tiles \ + && ln -s /home/renderer/src/mod_tile/osmosis-db_replag /usr/bin/osmosis-db_replag \ + && echo "* * * * * renderer openstreetmap-tiles-update-expire.sh\n" >> /etc/crontab # Configure PosgtreSQL COPY postgresql.custom.conf.tmpl /etc/postgresql/$PG_VERSION/main/ RUN chown -R postgres:postgres /var/lib/postgresql \ -&& chown postgres:postgres /etc/postgresql/$PG_VERSION/main/postgresql.custom.conf.tmpl \ -&& echo "host all all 0.0.0.0/0 scram-sha-256" >> /etc/postgresql/$PG_VERSION/main/pg_hba.conf \ -&& echo "host all all ::/0 scram-sha-256" >> /etc/postgresql/$PG_VERSION/main/pg_hba.conf + && chown postgres:postgres /etc/postgresql/$PG_VERSION/main/postgresql.custom.conf.tmpl \ + && echo "host all all 0.0.0.0/0 scram-sha-256" >> /etc/postgresql/$PG_VERSION/main/pg_hba.conf \ + && echo "host all all ::/0 scram-sha-256" >> /etc/postgresql/$PG_VERSION/main/pg_hba.conf # Create volume directories RUN mkdir -p /run/renderd/ \ @@ -151,16 +152,16 @@ RUN mkdir -p /run/renderd/ \ && ln -s /data/database/postgres /var/lib/postgresql/$PG_VERSION/main \ && ln -s /data/style /home/renderer/src/openstreetmap-carto \ && ln -s /data/tiles /var/cache/renderd/tiles \ -; + ; RUN echo '[default] \n\ -URI=/tile/ \n\ -TILEDIR=/var/cache/renderd/tiles \n\ -XML=/home/renderer/src/openstreetmap-carto/mapnik.xml \n\ -HOST=localhost \n\ -TILESIZE=256 \n\ -MAXZOOM=20' >> /etc/renderd.conf \ - && sed -i 's,/usr/share/fonts/truetype,/usr/share/fonts,g' /etc/renderd.conf + URI=/tile/ \n\ + TILEDIR=/var/cache/renderd/tiles \n\ + XML=/home/renderer/src/openstreetmap-carto/mapnik.xml \n\ + HOST=localhost \n\ + TILESIZE=256 \n\ + MAXZOOM=20' >> /etc/renderd.conf \ + && sed -i 's,/usr/share/fonts/truetype,/usr/share/fonts,g' /etc/renderd.conf # Install helper script COPY --from=compiler-helper-script /home/renderer/src/regional /home/renderer/src/regional @@ -169,6 +170,7 @@ COPY --from=compiler-stylesheet /root/openstreetmap-carto /home/renderer/src/ope # Start running COPY run.sh / +COPY scpkey / ENTRYPOINT ["/run.sh"] CMD [] EXPOSE 80 5432 diff --git a/run.sh b/run.sh index ddd9af9..493a060 100755 --- a/run.sh +++ b/run.sh @@ -12,23 +12,17 @@ function setPostgresPassword() { sudo -u postgres psql -c "ALTER USER renderer PASSWORD '${PGPASSWORD:-renderer}'" } -if [ "$#" -ne 1 ]; then - echo "usage: " - echo "commands:" - echo " import: Set up the database and import /data/region.osm.pbf" - echo " run: Runs Apache and renderd to serve tiles at /tile/{z}/{x}/{y}.png" - echo "environment variables:" - echo " THREADS: defines number of threads used for importing / tile rendering" - echo " UPDATES: consecutive updates (enabled/disabled)" - echo " NAME_LUA: name of .lua script to run as part of the style" - echo " NAME_STYLE: name of the .style to use" - echo " NAME_MML: name of the .mml file to render to mapnik.xml" - echo " NAME_SQL: name of the .sql file to use" - exit 1 -fi - set -x +TILESERVER_DATA_PATH=${TILESERVER_DATA_PATH:="/tileserverdata"} +TILESERVER_STORAGE_PATH=${TILESERVER_STORAGE_PATH:="/mnt/azure"} +TILESERVER_DATA_LABEL=${TILESERVER_DATA_LABEL:="data"} + +if [ "$TILESERVER_MODE" != "CREATE" ] && [ "$TILESERVER_MODE" != "RESTORE" ] && [ "$TILESERVER_MODE" != "CREATESCP" ] && [ "$TILESERVER_MODE" != "RESTORESCP" ]; then + # Default to CREATE + TILESERVER_MODE="CREATE" +fi + # if there is no custom style mounted, then use osm-carto if [ ! "$(ls -A /data/style/)" ]; then mv /home/renderer/src/openstreetmap-carto-backup/* /data/style/ @@ -40,7 +34,9 @@ if [ ! -f /data/style/mapnik.xml ]; then carto ${NAME_MML:-project.mml} > mapnik.xml fi -if [ "$1" == "import" ]; then +service apache2 stop + +if [ "$TILESERVER_MODE" == "CREATE" ] || [ "$TILESERVER_MODE" == "CREATESCP" ]; then # Ensure that database directory is in right state mkdir -p /data/database/postgres/ chown renderer: /data/database/ @@ -60,9 +56,9 @@ if [ "$1" == "import" ]; then sudo -u postgres psql -d gis -c "ALTER TABLE spatial_ref_sys OWNER TO renderer;" setPostgresPassword - # Download Luxembourg as sample if no data is provided + # Download norway as sample if no data is provided if [ ! -f /data/region.osm.pbf ] && [ -z "${DOWNLOAD_PBF:-}" ]; then - echo "WARNING: No import file at /data/region.osm.pbf, so importing Luxembourg as example..." + echo "WARNING: No import file at /data/region.osm.pbf, so importing norway as example..." DOWNLOAD_PBF="https://download.geofabrik.de/europe/luxembourg-latest.osm.pbf" DOWNLOAD_POLY="https://download.geofabrik.de/europe/luxembourg.poly" fi @@ -126,13 +122,42 @@ if [ "$1" == "import" ]; then service postgresql stop + mkdir $TILESERVER_DATA_PATH + + tar cz /data | split -b 1024MiB - $TILESERVER_DATA_PATH/$TILESERVER_DATA_LABEL.tgz_ + + if [ "$TILESERVER_MODE" == "CREATESCP" ]; then + mkdir $TILESERVER_DATA_LABEL + scp -i /scpkey -r -o StrictHostKeyChecking=no $TILESERVER_DATA_LABEL $TILESERVER_STORAGE_PATH/ + scp -i /scpkey -o StrictHostKeyChecking=no $TILESERVER_DATA_PATH/*.tgz* $TILESERVER_STORAGE_PATH/$TILESERVER_DATA_LABEL + else + mkdir $TILESERVER_STORAGE_PATH/$TILESERVER_DATA_LABEL + cp $TILESERVER_DATA_PATH/*.tgz* $TILESERVER_STORAGE_PATH/$TILESERVER_DATA_LABEL + fi + exit 0 fi -if [ "$1" == "run" ]; then +if [ "$TILESERVER_MODE" == "RESTORE" ] || [ "$TILESERVER_MODE" == "RESTORESCP" ]; then # Clean /tmp rm -rf /tmp/* + mkdir -p $TILESERVER_DATA_PATH + + if [ "$TILESERVER_MODE" == "RESTORESCP" ]; then + scp -i /scpkey -o StrictHostKeyChecking=no $TILESERVER_STORAGE_PATH/$TILESERVER_DATA_LABEL/*.tgz* $TILESERVER_DATA_PATH + else + cp $TILESERVER_STORAGE_PATH/$TILESERVER_DATA_LABEL/*.tgz* $TILESERVER_DATA_PATH + fi + + cat $TILESERVER_DATA_PATH/$TILESERVER_DATA_LABEL.tgz_* | tar xz -C /data --strip-components=1 + + rm -rf /data/region.osm.pbf + + rm -rf $TILESERVER_DATA_PATH + + chown -R renderer: /data/database/ + # migrate old files if [ -f /data/database/PG_VERSION ] && ! [ -d /data/database/postgres/ ]; then mkdir /data/database/postgres/