diff --git a/Dockerfile b/Dockerfile index b437be5..1e1fbeb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -139,7 +139,8 @@ RUN adduser --disabled-password --gecos "" renderer # Install python libraries RUN pip3 install \ requests \ - pyyaml + pyyaml \ + osmium # Configure Apache RUN mkdir /var/lib/mod_tile \ @@ -160,6 +161,8 @@ RUN chmod +x /usr/bin/openstreetmap-tiles-update-expire \ && mkdir /var/log/tiles \ && chmod a+rw /var/log/tiles \ && ln -s /home/renderer/src/mod_tile/osmosis-db_replag /usr/bin/osmosis-db_replag \ +&& echo "SHELL=/bin/bash" > /etc/crontab \ +&& echo "BASH_ENV=/container.env" >> /etc/crontab \ && echo "* * * * * renderer openstreetmap-tiles-update-expire\n" >> /etc/crontab RUN mkdir /nodes \ diff --git a/README.md b/README.md index 4d6f47b..9551c2f 100644 --- a/README.md +++ b/README.md @@ -29,9 +29,12 @@ If your import is an extract of the planet and has polygonal bounds associated w ``` docker run \ -e UPDATES=enabled \ + -e REPLICATION_URL=https://planet.openstreetmap.org/replication/minute/ \ + -e MAX_INTERVAL_SECONDS=60 \ -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf \ -v /absolute/path/to/luxembourg.poly:/data.poly \ -v openstreetmap-data:/var/lib/postgresql/12/main \ + -v openstreetmap-rendered-tiles:/var/lib/mod_tile \ overv/openstreetmap-tile-server \ import ``` @@ -92,6 +95,8 @@ Given that you've set up your import as described in the *Automatic updates* sec ``` docker run \ -p 8080:80 \ + -e REPLICATION_URL=https://planet.openstreetmap.org/replication/minute/ \ + -e MAX_INTERVAL_SECONDS=60 \ -e UPDATES=enabled \ -v openstreetmap-data:/var/lib/postgresql/12/main \ -v openstreetmap-rendered-tiles:/var/lib/mod_tile \ diff --git a/openstreetmap-tiles-update-expire b/openstreetmap-tiles-update-expire old mode 100644 new mode 100755 index 4a0e7e4..e752850 --- a/openstreetmap-tiles-update-expire +++ b/openstreetmap-tiles-update-expire @@ -30,7 +30,6 @@ OSM2PGSQL_OPTIONS="-d $DBNAME -G --hstore --tag-transform-script /home/renderer/ #------------------------------------------------------------------------------ TRIM_POLY_FILE="/var/lib/mod_tile/data.poly" TRIM_OPTIONS="-d $DBNAME" -#TRIM_REGION_OPTIONS="-b -14.17 48.85 2.12 61.27" TRIM_REGION_OPTIONS="-p $TRIM_POLY_FILE" BASE_DIR=/var/lib/mod_tile @@ -60,6 +59,10 @@ EXPIRY_TOUCHFROM=13 EXPIRY_DELETEFROM=19 EXPIRY_MAXZOOM=20 + +REPLICATION_URL=${REPLICATION_URL:="https://planet.openstreetmap.org/replication/hour/"} +MAX_INTERVAL_SECONDS=${MAX_INTERVAL_SECONDS:="3600"} + #************************************************************************* #************************************************************************* @@ -107,11 +110,18 @@ freelock() if [ $# -eq 1 ] ; then m_info "Initialising Osmosis replication system to $1" - mkdir $WORKOSM_DIR - $OSMOSIS_BIN --read-replication-interval-init workingDirectory=$WORKOSM_DIR 1>&2 2> "$OSMOSISLOG" - wget "https://replicate-sequences.osm.mazdermind.de/?"$1"T00:00:00Z" -O $WORKOSM_DIR/state.txt - mv $WORKOSM_DIR/configuration.txt $WORKOSM_DIR/configuration_orig.txt - sed "s!baseUrl=http://planet.openstreetmap.org/replication/minute!baseUrl=https://planet.openstreetmap.org/replication/minute!" $WORKOSM_DIR/configuration_orig.txt > $WORKOSM_DIR/configuration.txt + mkdir -p $WORKOSM_DIR + $OSMOSIS_BIN -v 5 --read-replication-interval-init workingDirectory=$WORKOSM_DIR 1>&2 2> "$OSMOSISLOG" + + init_seq=$(pyosmium-get-changes --server $REPLICATION_URL -D $1) + url_dynamicPart=$(printf %09d $init_seq | sed 's_\([0-9][0-9][0-9]\)\([0-9][0-9][0-9]\)\([0-9][0-9][0-9]\)_\1/\2/\3_') + wget $REPLICATION_URL/$url_dynamicPart.state.txt -O $WORKOSM_DIR/state.txt + + cat > /var/lib/mod_tile/.osmosis/configuration.txt <<- EOM +baseUrl=$REPLICATION_URL +maxInterval=$MAX_INTERVAL_SECONDS +EOM + else # make sure the lockfile is removed when we exit and then claim it @@ -142,7 +152,7 @@ fi /bin/cp $WORKOSM_DIR/state.txt $WORKOSM_DIR/last.state.txt m_ok "downloading diff" - if ! $OSMOSIS_BIN --read-replication-interval workingDirectory=$WORKOSM_DIR --simplify-change --write-xml-change $CHANGE_FILE 1>&2 2> "$OSMOSISLOG"; then + if ! $OSMOSIS_BIN -v 5 --read-replication-interval workingDirectory=$WORKOSM_DIR --simplify-change --write-xml-change $CHANGE_FILE 1>&2 2> "$OSMOSISLOG"; then m_error "Osmosis error" fi @@ -167,12 +177,12 @@ fi #------------------------------------------------------------------------------ # The lockfile is normally removed before we expire tiles because that is -# something thatcan be done in parallel with further processing. In order to +# something that can be done in parallel with further processing. In order to # avoid rework, if actually rerendering is done rather than just deleting or # dirtying, it makes sense to move it lower down. #------------------------------------------------------------------------------ -# m_ok "Import complete; removing lock file" -# freelock "$LOCK_FILE" +# m_ok "Import complete; removing lock file" +# freelock "$LOCK_FILE" m_ok "expiring tiles" #------------------------------------------------------------------------------ @@ -194,11 +204,8 @@ fi # Only remove the lock file after expiry (if system is slow we want to delay # the next import, not have multiple render_expired processes running) #------------------------------------------------------------------------------ - freelock "$LOCK_FILE" + freelock "$LOCK_FILE" m_ok "Done with import" - - - fi diff --git a/run.sh b/run.sh index 454239d..9bb175a 100755 --- a/run.sh +++ b/run.sh @@ -2,6 +2,8 @@ set -euo pipefail +declare -p | grep -Ev 'BASHOPTS|BASH_VERSINFO|EUID|PPID|SHELLOPTS|UID' > /container.env + function createPostgresConfig() { cp /etc/postgresql/12/main/postgresql.custom.conf.tmpl /etc/postgresql/12/main/conf.d/postgresql.custom.conf sudo -u postgres echo "autovacuum = $AUTOVACUUM" >> /etc/postgresql/12/main/conf.d/postgresql.custom.conf @@ -62,11 +64,12 @@ if [ "$1" = "import" ]; then if [ "${UPDATES:-}" = "enabled" ]; then # determine and set osmosis_replication_timestamp (for consecutive updates) osmium fileinfo /data.osm.pbf > /var/lib/mod_tile/data.osm.pbf.info - osmium fileinfo /data.osm.pbf | grep 'osmosis_replication_timestamp=' | cut -b35-44 > /var/lib/mod_tile/replication_timestamp.txt + osmium fileinfo --get=header.option.timestamp /data.osm.pbf > /var/lib/mod_tile/replication_timestamp.txt REPLICATION_TIMESTAMP=$(cat /var/lib/mod_tile/replication_timestamp.txt) # initial setup of osmosis workspace (for consecutive updates) - sudo -u renderer openstreetmap-tiles-update-expire $REPLICATION_TIMESTAMP + sudo -E -u renderer openstreetmap-tiles-update-expire $REPLICATION_TIMESTAMP + fi # copy polygon file if available