Debian Codegraph @

utils/openstreetmap-tiles-update-rerender

raw
1 #!/bin/sh
2 #
3 # Copyright (c) 2007 - 2023 by mod_tile contributors (see AUTHORS file)
4 #
5 # This program is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU General Public License
7 # as published by the Free Software Foundation; either version 2
8 # of the License, or (at your option) any later version.
9 #
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
14 #
15 # You should have received a copy of the GNU General Public License
16 # along with this program; If not, see http://www.gnu.org/licenses/.
17
18 set -e
19
20 #------------------------------------------------------------------------------
21 # AJT - change directory to mod_tile directory so that we can run replag
22 # and other things directly from this script when run from cron.
23 # Change the actual location to wherever installed locally.
24 #------------------------------------------------------------------------------
25 export PATH=.:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
26 ACCOUNT=renderaccount
27 cd /home/$ACCOUNT/src/mod_tile/
28
29 #------------------------------------------------------------------------------
30 # Extra OSM2PGSQL_OPTIONS may need setting if a tag transform script is
31 # in use. See https://github.com/SomeoneElseOSM/SomeoneElse-style and
32 # http://wiki.openstreetmap.org/wiki/User:SomeoneElse/Ubuntu_1404_tileserver_load
33 # The database name always needs setting.
34 #------------------------------------------------------------------------------
35 OSMOSIS_BIN=osmosis
36 OSM2PGSQL_BIN=osm2pgsql
37 TRIM_BIN=/home/$ACCOUNT/src/regional/trim_osc.py
38
39 DBNAME=gis
40 OSM2PGSQL_OPTIONS="-d $DBNAME"
41 #------------------------------------------------------------------------------
42 # When using trim_osc.py we can define either a bounding box (such as this
43 # example for England and Wales) or a polygon.
44 # See https://github.com/zverik/regional .
45 # This area will usually correspond to the data originally loaded.
46 #------------------------------------------------------------------------------
47 TRIM_OPTIONS="-d $DBNAME"
48 TRIM_REGION_OPTIONS="-b -14.17 48.85 2.12 61.27"
49 #TRIM_REGION_OPTIONS="-p region.poly"
50
51 BASE_DIR=/var/lib/mod_tile
52 LOG_DIR=/var/log/tiles/
53 WORKOSM_DIR=$BASE_DIR/.osmosis
54
55 LOCK_FILE=/tmp/openstreetmap-update-expire-lock.txt
56 CHANGE_FILE=$BASE_DIR/changes.osc.gz
57 EXPIRY_FILE=$BASE_DIR/dirty_tiles
58 STOP_FILE=$BASE_DIR/stop.txt
59
60 OSMOSISLOG=$LOG_DIR/osmosis.log
61 PGSQLLOG=$LOG_DIR/osm2pgsql.log
62 EXPIRYLOG=$LOG_DIR/expiry.log
63 RUNLOG=$LOG_DIR/run.log
64
65 #------------------------------------------------------------------------------
66 # The tile expiry section below can re-render, delete or dirty expired tiles.
67 # By default, tiles between EXPIRY_MINZOOM and EXPIRY_MAXZOOM are rerendered.
68 # "render_expired" can optionally delete (and/or dirty) tiles above a certail
69 # threshold rather than rendering them.
70 # Here we rerender tiles between zoom levels 13 and 18 and delete between 19
71 # and 20.
72 #------------------------------------------------------------------------------
73 EXPIRY_MINZOOM=13
74 EXPIRY_TOUCHFROM=21
75 EXPIRY_DELETEFROM=19
76 EXPIRY_MAXZOOM=20
77
78 #*************************************************************************
79 #*************************************************************************
80
81 m_info() {
82 echo "[$(date +"%Y-%m-%d %H:%M:%S")] $$ $1" >>"$RUNLOG"
83 }
84
85 m_error() {
86 echo "[$(date +"%Y-%m-%d %H:%M:%S")] $$ [error] $1" >>"$RUNLOG"
87
88 m_info "resetting state"
89 /bin/cp $WORKOSM_DIR/last.state.txt $WORKOSM_DIR/state.txt || true
90
91 rm "$CHANGE_FILE" || true
92 rm "$EXPIRY_FILE.$$" || true
93 rm "$LOCK_FILE"
94 exit
95 }
96
97 m_ok() {
98 echo "[$(date +"%Y-%m-%d %H:%M:%S")] $$ $1" >>"$RUNLOG"
99 }
100
101 getlock() {
102 if [ -s $1 ]; then
103 if [ "$(ps -p $(cat $1) | wc -l)" -gt 1 ]; then
104 return 1 #false
105 fi
106 fi
107
108 echo $$ >"$1"
109 return 0 #true
110 }
111
112 freelock() {
113 rm "$1"
114 rm "$CHANGE_FILE"
115 }
116
117 if [ $# -eq 1 ]; then
118 m_info "Initialising Osmosis replication system to $1"
119 mkdir $WORKOSM_DIR
120 $OSMOSIS_BIN --read-replication-interval-init workingDirectory=$WORKOSM_DIR 1>&2 2>"$OSMOSISLOG"
121 wget "http://replicate-sequences.osm.mazdermind.de/?"$1"T00:00:00Z" -O $WORKOSM_DIR/state.txt
122 mv $WORKOSM_DIR/configuration.txt $WORKOSM_DIR/configuration_orig.txt
123 sed "s!baseUrl=http://planet.openstreetmap.org/replication/minute!baseUrl=https://planet.openstreetmap.org/replication/minute!" $WORKOSM_DIR/configuration_orig.txt >$WORKOSM_DIR/configuration.txt
124 else
125 # make sure the lockfile is removed when we exit and then claim it
126
127 if ! getlock "$LOCK_FILE"; then
128 m_info "pid $(cat $LOCK_FILE) still running"
129 exit 3
130 fi
131
132 if [ -e $STOP_FILE ]; then
133 m_info "stopped"
134 exit 2
135 fi
136
137 # -----------------------------------------------------------------------------
138 # Add disk space check from https://github.com/zverik/regional
139 # -----------------------------------------------------------------------------
140 MIN_DISK_SPACE_MB=500
141
142 if $(python -c "import os, sys; st=os.statvfs('$BASE_DIR'); sys.exit(1 if st.f_bavail*st.f_frsize/1024/1024 > $MIN_DISK_SPACE_MB else 0)"); then
143 m_info "there is less than $MIN_DISK_SPACE_MB MB left"
144 exit 4
145 fi
146
147 seq=$(cat $WORKOSM_DIR/state.txt | grep sequenceNumber | cut -d= -f2)
148
149 m_ok "start import from seq-nr $seq, replag is $(osmosis-db_replag -h)"
150
151 /bin/cp $WORKOSM_DIR/state.txt $WORKOSM_DIR/last.state.txt
152 m_ok "downloading diff"
153
154 if ! $OSMOSIS_BIN --read-replication-interval workingDirectory=$WORKOSM_DIR --simplify-change --write-xml-change $CHANGE_FILE 1>&2 2>"$OSMOSISLOG"; then
155 m_error "Osmosis error"
156 fi
157
158 m_ok "filtering diff"
159 if ! $TRIM_BIN $TRIM_OPTIONS $TRIM_REGION_OPTIONS -z $CHANGE_FILE $CHANGE_FILE 1>&2 2>>"$RUNLOG"; then
160 m_error "Trim_osc error"
161 fi
162
163 m_ok "importing diff"
164 #------------------------------------------------------------------------------
165 # Previously openstreetmap-tiles-update-expire tried to dirty layer
166 # "$EXPIRY_MAXZOOM - 3" (which was 15) only. Instead we write all expired
167 # tiles in range to the list (note the "-" rather than ":" in the "-e"
168 # parameter).
169 #------------------------------------------------------------------------------
170 if ! $OSM2PGSQL_BIN -a --slim -e$EXPIRY_MINZOOM-$EXPIRY_MAXZOOM $OSM2PGSQL_OPTIONS -o "$EXPIRY_FILE.$$" $CHANGE_FILE 1>&2 2>"$PGSQLLOG"; then
171 m_error "osm2pgsql error"
172 fi
173
174 #------------------------------------------------------------------------------
175 # The lockfile is normally removed before we expire tiles because that is
176 # something thatcan be done in parallel with further processing. In order to
177 # avoid rework, if actually rerendering is done rather than just deleting or
178 # dirtying, it makes sense to move it lower down.
179 #------------------------------------------------------------------------------
180 # m_ok "Import complete; removing lock file"
181 # freelock "$LOCK_FILE"
182
183 m_ok "expiring tiles"
184 #------------------------------------------------------------------------------
185 # When expiring tiles we need to define the style sheet if it's not "default".
186 # In this case it's "ajt".
187 # Previously all tiles on the "dirty" list between $EXPIRY_MINZOOM and
188 # $EXPIRY_MAXZOOM were dirtied. We currently re-render
189 # tiles >= $EXPIRY_MINZOOM and < $EXPIRY_DELETEFROM, expiry from 14 and
190 # delete >= $EXPIRY_DELETEFROM and <= $EXPIRY_MAXZOOM.
191 # The default path to renderd.sock is fixed.
192 #
193 # Note no --touch-from=$EXPIRY_TOUCHFROM
194 #------------------------------------------------------------------------------
195 if ! render_expired --map=ajt --min-zoom=$EXPIRY_MINZOOM --delete-from=$EXPIRY_DELETEFROM --max-zoom=$EXPIRY_MAXZOOM -s /run/renderd/renderd.sock <"$EXPIRY_FILE.$$" 2>&1 | tail -8 >>"$EXPIRYLOG"; then
196 m_info "Expiry failed"
197 fi
198
199 rm "$EXPIRY_FILE.$$"
200
201 #------------------------------------------------------------------------------
202 # Only remove the lock file after expiry (if system is slow we want to delay
203 # the next import, not have multiple render_expired processes running)
204 #------------------------------------------------------------------------------
205 freelock "$LOCK_FILE"
206
207 m_ok "Done with import"
208
209 fi