Server live flight tracker. How to refresh data , with 2nd callback loop?

Hello, I’m learning bokeh and trying the server to live track flights. It works well, tracking dozens of planes overhead for 30 seconds.

But I’m trying to auto refresh (clean) the map after say 30s to remove the old tracks. Ive tried asyncio and looked at sessions without success.

It seems I should be able to use a nested periodic_callbacks, see code below which uses add_periodic_callback to refresh existing flights every few seconds.

Is there a way to ‘nest’ calbacks?
or maybe another method to refresh the data?

Thanks for any guidance.

    from bokeh.plotting import figure, curdoc #show, output_file, save
    from bokeh.tile_providers import get_provider, Vendors
    from bokeh.models import ColumnDataSource, LabelSet
    from opensky_api import OpenSkyApi
    from pyproj import Transformer, CRS
    from bokeh.driving import linear
    trans = Transformer.from_crs(4326, 3857)
    api = OpenSkyApi(username='********', password='*********')
    import time

    class Update(object):
        """ # epsg.io WGS 84 (46.515229, 6.559460) - EPSG 3857 (730195.75, 5863302.96)
        get flight stats. bbox and extent are reversed lat/lon. d = scale factor
        USE PROJ FOR TUPLS GEODESIC TO CRS
        {   'baro_altitude': 10660.38,     'callsign': 'DLH02P  ',
        'geo_altitude': 11049,     'heading': 200.38,
        'icao24': '3c664e',    'last_contact': 1567365699,
        'latitude': 45.5358,    'longitude': 6.1761,
        'on_ground': False,    'origin_country': 'Germany',
        'position_source': 0,    'sensors': None,
        'spi': False,    'squawk': '2504',
        'time_position': 1567365699,    'velocity': 220.07,
        'vertical_rate': 0}
        """
        d = 0.01# size of map 0.01 about 1° square
        def __init__(self, loc):
            self.loc=loc
            self.lat,self.lon = locations[self.loc]
            self.states = api.get_states(bbox=(self.get_bbox())).states
            self.ss=[] # callsigns
            self.sslat=[] # list of lats and longs
            self.sslon=[]
            self.ssorig=[]
            self.ssalt=[]
            self.ssair=[]
            self.ssv=[]
            self.ssvr=[]
            self.ssh=[]
            if len(self.states) == 0:
                print('nothing found')
            else:
                for s in self.states:
                    #print('{} {},V{}m/s'.format(s.callsign,s.heading, s.velocity))
                    self.east,self.north = self.lalo_en(s.latitude,s.longitude)
                    #self.ss.update({s.callsign:[s.longitude, s.latitude, s.baro_altitude, s.velocity, s.callsign, s.origin_country]})
                    self.ss.append(s.callsign)
                    self.sslon.append(self.east)
                    self.sslat.append(self.north)
                    self.ssorig.append(s.origin_country)
                    self.ssalt.append(s.baro_altitude)
                    self.ssair.append(s.on_ground) # False = air
                    self.ssv.append(self.speed(s.velocity))
                    self.ssvr.append(s.vertical_rate)
                    self.ssh.append(s.heading)

        def speed(self, v): # m/s to Km/hr 3600/1000
            return v * 3.6

        def get_yrange(self): # north   # x (mlon) tuple min,max = extent of the map blokeh plot
            east,north = self.lalo_en(self.lat, self.lon)
            return (north - north*Update.d , north + north*Update.d)

        def get_xrange(self): # east   # y (lat) tuple min,max = extent of the map blokeh plot
            east,north = self.lalo_en(self.lat, self.lon)
            return (east - east*Update.d , east + east*Update.d)

        def get_bbox(self):        # tuple (min latitude, max latitude, min longitude, max longitude)
            return (self.lat - 100*Update.d, self.lat + 100*Update.d, self.lon - 100*Update.d , self.lon + 100*Update.d)

        def lalo_en(self, lat,lon):
            east, north = trans.transform(lat,lon) # Lat, Lon -> X East, Y North
            return east,north

    locations = {'BKK': (13.69269, 100.750465 ), 'HOM': (46.515229, 6.559460), 'GVA': (46.23631, 6.108055)}
    location = 'HOM'
    TOOLTIPS = [("Origin", "@orig"), ("Alt m", "@alt{0.0a}"),("Speed km/h", "@vel{0}"), ("Dir", "@head"), ("Climb m/s", "@vr")]

    a = Update(location)
    tile_provider = get_provider(Vendors.CARTODBPOSITRON)  # STAMEN_TERRAIN)
    source = ColumnDataSource(data=dict(lon=a.sslon, lat=a.sslat,
                                        cs=a.ss,orig=a.ssorig,
                                        alt=a.ssalt,air=a.ssair,
                                        vel=a.ssv, vr=a.ssvr,
                                        head=a.ssh))

    p = figure(x_range=a.get_xrange(), y_range=a.get_yrange(), x_axis_type="mercator", y_axis_type="mercator", tooltips=TOOLTIPS)
    p.add_tile(tile_provider)
    labels1 = LabelSet(x='lon', y='lat', text='cs', x_offset=1, y_offset=1, source=source,text_font_size='8pt')
    p.circle(x='lon', y='lat', size=5, fill_color="red", fill_alpha=0.8, source=source)
    p.square(x=a.lalo_en(a.lat,a.lon)[0], y=a.lalo_en(a.lat,a.lon)[1], size=10, fill_color="blue", fill_alpha=0.8)
    p.circle(x='lon', y='lat', size=6, fill_color="red", fill_alpha=0.8, source=source)
    p.add_layout(labels1)

    @linear()
    def steps(value):
        print(f'step count {value}')
        if value <5:
            b = Update(location)
            bsource.stream({"lon":b.sslon, "lat":b.sslat})
            p.circle(x='lon', y='lat', size=4, fill_color="red", fill_alpha=0.8, source=bsource)
        else:
            curdoc().remove_periodic_callback(id)

    time.sleep(5)
    bsource = ColumnDataSource({"lon":[], "lat":[]})
    curdoc().add_root(p)
    curdoc().title="current satus"
    # Add a periodic callback to be run every 5001 milliseconds
    id = curdoc().add_periodic_callback(steps, 6001)
    """

There’s alot of missing context here. I am going to assume that this external API you are hitting is synchronous, as it appears to be from the code. In which case, if you want to, e.g.

  • update every 5 seconds
  • also clear old data every 30 seconds

Then I would suggest just having a single callback that runs every 5 seconds, and that also happens to clear data every 6th invocation. You can use a decorator from bokeh.driving on your callback to help arrange the repeated counting that conditions the clear operation.

If that’s not what you are wanting, then some additional description is necessary.

Thanks Bryan,
You’re correct in the API provides updates at minimum 5 sec intervals.
So, I already use your suggested bokeh.driving with the @linear decorator to help count that inner loop:-

@linear()
def steps(value):
print(f’step count {value}’)
if value <5:
b = Update(location)
bsource.stream({“lon”:b.sslon, “lat”:b.sslat})
p.circle(x=‘lon’, y=‘lat’, size=4, fill_color=“red”, fill_alpha=0.8, source=bsource)
else:
curdoc().remove_periodic_callback(id)

My issue is how the clear the old data, since “curdoc().remove_periodic_callback(id)” just removes it but doesn’t restart the outer loop.

@awsbarker I am not sure why you want to remove the callback? My understanding is that you want things to update continuously forever, in which case the same callback should run forever. Instead of value < 5 you should do something like value % 6 == 0 which will be True every sixth iteration. Alternatively you could use the @repeat(range(6)) decorator instead, which just repeats endlessly, then check value == 5.

Also you don’t need to (and almost certainly should not) call p.circle at all in your callback. That is for sure to leak resources over time (and is also not going to to what you want, either). You should call p.circle once (outside the callback) and then only update the data source for the circle in the callback.

BTW you can also supply a rollover parameter to stream, which will automatically drop old data points after the threshold you specify. In which case maybe your callback doesn’t need all this “reset every sixth time” logic at all.

Thanks again Bryan,

I solved it by retaining single callback and tweaking the rollovers in the two data streams with different and dynamic values according to the number of planes - see below relevant code below.

I removed the p.circles inside callback as you suggested so all setup is done outside callback.

I’d never seen value % 6 == 0 before so stuck with if value == 5 but used your suggested @repeat(range(6)

Works great, so now to stream it to existing django site…
I’m so happy with bokeh results I’m going to revisit some of my older plot and ‘upgrade’ via bokeh.

Andrew

@repeat(range(6))
def steps(value):
# rolloverb= 6 x planes, rollover = planes
b = Update(location)
rob = len(b.ss) * 5
print(f’step count {value}. tracker points= {rob}’)
bsource.stream({“lon”:b.sslon, “lat”:b.sslat}, rollover=rob)
if value == 5:
print(f’updated {len(b.ss)} planes ')
source.stream({“lon”:b.sslon,“lat”:b.sslat,“cs”:b.ss,“orig”:b.ssorig,
“alt”:b.ssalt,“air”:b.ssair,“vel”:b.ssv,“vr”:b.ssvr, “head”:b.ssh}, rollover=len(b.ss))
bsource.stream({“lon”: b.sslon, “lat”: b.sslat}, rollover=len(b.ss))

trans = Transformer.from_crs(4326, 3857)
api = OpenSkyApi(username=**, password=password)
locations = {‘BKK’: (13.69269, 100.750465 ), ‘HOM’: (46.5229, 6.559460), ‘GVA’: (46.23631, 6.108055)}
location = ‘HOM’
TOOLTIPS = [(“Origin”, “@orig”), (“Alt m”, “@alt{0.0a}”),(“Speed km/h”, “@vel{0}”), (“Dir”, “@head”), (“Climb m/s”, “@vr”)]

a = Update(location)
tile_provider = get_provider(Vendors.CARTODBPOSITRON) # STAMEN_TERRAIN)
source = ColumnDataSource(data=dict(lon=a.sslon, lat=a.sslat, cs=a.ss,orig=a.ssorig,
alt=a.ssalt,air=a.ssair, vel=a.ssv, vr=a.ssvr, head=a.ssh))
p = figure(x_range=a.get_xrange(), y_range=a.get_yrange(), x_axis_type=“mercator”, y_axis_type=“mercator”, tooltips=TOOLTIPS)
p.add_tile(tile_provider)
labels1 = LabelSet(x=‘lon’, y=‘lat’, text=‘cs’, x_offset=1, y_offset=1, source=source,text_font_size=‘8pt’)
p.square(x=a.lalo_en(a.lat,a.lon)[0], y=a.lalo_en(a.lat,a.lon)[1], size=10, fill_color=“blue”, fill_alpha=0.8)
p.circle(x=‘lon’, y=‘lat’, size=10, fill_color=“red”, fill_alpha=0.8, source=source)
p.add_layout(labels1)

bsource = ColumnDataSource({“lon”:, “lat”:})
p.circle(x=‘lon’, y=‘lat’, size=4, fill_color=“red”, fill_alpha=0.8, source=bsource)
curdoc().add_root§
curdoc().title=f"Flights Status {location}"

Add a periodic callback to be run every 5001 milliseconds

curdoc().add_periodic_callback(steps, 5500)

1 Like

Hi Again Bryan,

I tried to get everything going behind a apache reverse proxy which worked using documented notes for http but the websockets for https just didn’t quiet work, seems might be buffer issue, and I can see from forum this is unresolved.

I think it would be important model to provide Bokeh mixed content (https - http) behind an apache reverse proxy, so if I do resolve it I’ll feedback here. OK?

FWIW the next version of Bokeh (1.4) will be able to terminate SSL directly, without a proxy. Regardless, I agree it would great to have guidance for running Bokeh behind an Apache proxy with SSL enabled. I don’t have any experience with Apache tho (and no bandwidth to spend on learning) so if you figure that out it would be great to report back. I’d suggest a brand new topic in that case.

Hello Bryan, I’m sad to report that after a month of trying I’m unable to confirm a complete solution.

It seems to be a websockets issue, ws are required and a straight reverse proxy like your docs seems to fail.

I did get this setup to work on LAN using several browsers (chrome, firefox, eolie) on several devices (linux, android):-

<VirtualHost *:443>
DocumentRoot /var/www/html/awsb.ddns.net/
ServerName awsb.ddns.net
ServerAdmin [email protected]

SSLEngine on	
SSLProxyEngine on
SSLProxyVerify None
#SSLProxyVerifyDepth 1
SSLProxyCheckPeerCN Off
SSLProxyCheckPeerName Off
SSLProxyCheckPeerExpire Off

SSLCertificateFile /etc/letsencrypt/archive/awsb.ddns.net/fullchain1.pem
SSLCertificateKeyFile /etc/letsencrypt/archive/awsb.ddns.net/privkey1.pem
SSLProtocol    all -SSLv2 -SSLv3 
SSLProxyProtocol +TLSv1 +TLSv1.1 +TLSv1.2 
SSLHonorCipherOrder     On
Include /etc/letsencrypt/options-ssl-apache.conf
SSLCipherSuite HIGH:!RC4:!MD5:!aNULL:!eNULL:!NULL:!DH:!EDH:!EXP:+MEDIUM

ProxyPreserveHost On
ProxyRequests off

 <Proxy *>
	Require all granted
	Options None
 </Proxy>            

Header always set Strict-Transport-Security "max-age=31536000"
RequestHeader set X-Forwarded-Proto http
RequestHeader set X-Forwarded-Port 443
RequestHeader set X-SCHEME http

    RewriteEngine On
    RewriteCond %{HTTP:Upgrade} websocket [NC]
RewriteCond %{HTTP:Connection} upgrade [NC]
RewriteCond %{HTTP:Connection} ^keep-alive,\ upgrade$ [NC]
RewriteRule ^/flights(.*)$    ws://127.0.0.1:5006/flights$1 [P]
    ProxyPassReverse /flights http://127.0.0.1:5006/

I suppose further testing by more experienced people may help.