Bokeh updates slow down tremendously after a couple hours (live Plotting)

Hi everyone.

I have a graph I built using Bokeh that updates the graph (which contains multiple linegraphs) every time a notify message is sent from one of the Observable classes. The data point set (which includes a Key (which is the name of the line graph to update a timestamp and a value) is plot every time notify is called. for the first 2 hours the plot points are added seamlessly and the graph updates very quickly. But after that point the updates are trailing the notify message by a couple hours. (i.e. when i’m streaming time (14:50:18.518, val 89.000) its only graphing 10:51:18.532, val 76 - this is after 8 hours of streaming data in). I need a days worth of data to be shown which is why my stream buffer is 345600. The weird thing that confuses me is that the plot continue to update data from a couple hours ago, even though its a couple hours behind the real time inputs… my guess is that there is a data buffer that is pushing the values onto the graph.

Is there anything I am doing wrong? is there anyway to speed up the process? workarounds? Would down sampling possibly help? Thanks for your help!

Here is my code.
class LiveGrapher: #Observer

def __init__(self, observable):
    self.datasetsDisplayed = observable.registerObserver(self)
    self.source = {}
    self.colors = ['red', 'green', 'blue', 'black', 'purple', 'brown','AliceBlue', 'AniqueWhite', 'Aqua', 'Aquamarine', 'Azure', 'Beige', 'Bisque']
    for item in self.datasetsDisplayed:
        self.source[item] = ColumnDataSource(data=dict(x=[], y=[], dateStampStr = [], timeStampStr = [], key = []))
       
    self.doc = curdoc()  

def notify(self, observable, *args, **kwargs):
    timeStamp = ''
    YKey = ''
    YVal = 0
    timeStamp = kwargs['timeStamp']
    for key in kwargs.keys():
        if key in self.datasetsDisplayed:
           YKey = key
           YVal = kwargs[YKey]
           self.doc.add_next_tick_callback(partial(self.update, x=timeStamp, y=YVal, key =YKey))
          
   

def update(self, x, y, key):
    self.source[key].stream(dict(x=[x], y=[y], key = [key], dateStampStr=[x.strftime('%m/%d/%Y')], timeStampStr=[x.strftime('%H:%M:%S')]), 345600)

def draw(self):
    drawTools = [BoxZoomTool(), ResetTool(), HoverTool( tooltips=[('key', '@key'), ('date', '@dateStampStr'), ('time', '@timeStampStr'), ('value' , '@y')])]
   
    p = figure(x_axis_type="datetime",  y_range=[0,2000], tools=drawTools, plot_width=1000)
   
    colorID = 0
    for dataset in self.datasetsDisplayed:
        p.line(x='x', y='y', legend = dataset, source=self.source[dataset], line_color=self.colors[colorID])
        #p.square(x='x', y='y', legend= dataset, source=self.source[dataset], fill_color=None, line_color=self.colors[colorID])
        colorID = colorID + 1
    p.legend.location = "top_right"
    p.legend.background_fill_color = "grey"
    p.legend.background_fill_alpha = 0.5
   

    self.doc.add_root(p)

So i stripped down my code so that it updates the x position every 0.01 seconds and it appears that what the graph displays and my actual count starts to differ at around ~ 15000 mark (which was right around the point my code above started to slow down (71 minutes in, displaying about 15000 points). Is it a pure drawing issue?

YVal = 200

XVal = 0

source = ColumnDataSource(data=dict(x=, y=))

doc = curdoc()

def notify():

global YVal, XVal, doc

while True:

print “i am here in notify”

XVal = XVal + 1

time.sleep(0.01)

print XVal

doc.add_next_tick_callback(partial(update, x=XVal, y=YVal))

def update( x, y):

print “i am here in update”

global source

source.stream(dict(x=, y=[y]), 345600)

p = figure(y_range=[0,1000], plot_width=1000)

p.line(x=‘x’, y=‘y’, source=source)

doc.add_root(p)

thread = Thread(target = notify)

thread.start()

bokeh serve --port 51

···

On Wednesday, 18 January 2017 10:05:16 UTC-5, Eugene Cheung wrote:

Hi everyone.

I have a graph I built using Bokeh that updates the graph (which contains multiple linegraphs) every time a notify message is sent from one of the Observable classes. The data point set (which includes a Key (which is the name of the line graph to update a timestamp and a value) is plot every time notify is called. for the first 2 hours the plot points are added seamlessly and the graph updates very quickly. But after that point the updates are trailing the notify message by a couple hours. (i.e. when i’m streaming time (14:50:18.518, val 89.000) its only graphing 10:51:18.532, val 76 - this is after 8 hours of streaming data in). I need a days worth of data to be shown which is why my stream buffer is 345600. The weird thing that confuses me is that the plot continue to update data from a couple hours ago, even though its a couple hours behind the real time inputs… my guess is that there is a data buffer that is pushing the values onto the graph.

Is there anything I am doing wrong? is there anyway to speed up the process? workarounds? Would down sampling possibly help? Thanks for your help!

Here is my code.
class LiveGrapher: #Observer

def __init__(self, observable):
    self.datasetsDisplayed = observable.registerObserver(self)
    self.source = {}
    self.colors = ['red', 'green', 'blue', 'black', 'purple', 'brown','AliceBlue', 'AniqueWhite', 'Aqua', 'Aquamarine', 'Azure', 'Beige', 'Bisque']
    for item in self.datasetsDisplayed:
        self.source[item] = ColumnDataSource(data=dict(x=[], y=[], dateStampStr = [], timeStampStr = [], key = []))
       
    self.doc = curdoc()  

def notify(self, observable, *args, **kwargs):
    timeStamp = ''
    YKey = ''
    YVal = 0
    timeStamp = kwargs['timeStamp']
    for key in kwargs.keys():
        if key in self.datasetsDisplayed:
           YKey = key
           YVal = kwargs[YKey]
           self.doc.add_next_tick_callback(partial(self.update, x=timeStamp, y=YVal, key =YKey))
          
   

def update(self, x, y, key):
    self.source[key].stream(dict(x=[x], y=[y], key = [key], dateStampStr=[x.strftime('%m/%d/%Y')], timeStampStr=[x.strftime('%H:%M:%S')]), 345600)

def draw(self):
    drawTools = [BoxZoomTool(), ResetTool(), HoverTool( tooltips=[('key', '@key'), ('date', '@dateStampStr'), ('time', '@timeStampStr'), ('value' , '@y')])]
   
    p = figure(x_axis_type="datetime",  y_range=[0,2000], tools=drawTools, plot_width=1000)
   
    colorID = 0
    for dataset in self.datasetsDisplayed:
        p.line(x='x', y='y', legend = dataset, source=self.source[dataset], line_color=self.colors[colorID])
        #p.square(x='x', y='y', legend= dataset, source=self.source[dataset], fill_color=None, line_color=self.colors[colorID])
        colorID = colorID + 1
    p.legend.location = "top_right"
    p.legend.background_fill_color = "grey"
    p.legend.background_fill_alpha = 0.5
   

    self.doc.add_root(p)

It's possible. There is a webgl option you could try. I believe it may currently have some regressions on safari, but for anything over a few tens of thousands of points, it's probably worth considering.

Thanks,

Bryan

···

On Jan 18, 2017, at 11:34 AM, Eugene Cheung <[email protected]> wrote:

So i stripped down my code so that it updates the x position every 0.01 seconds and it appears that what the graph displays and my actual count starts to differ at around ~ 15000 mark (which was right around the point my code above started to slow down (71 minutes in, displaying about 15000 points). Is it a pure drawing issue?
YVal = 200
XVal = 0
source = ColumnDataSource(data=dict(x=, y=))
doc = curdoc()
def notify():
    global YVal, XVal, doc
    while True:
        print "i am here in notify"
        XVal = XVal + 1
        time.sleep(0.01)
        print XVal
        doc.add_next_tick_callback(partial(update, x=XVal, y=YVal))
def update( x, y):
    print "i am here in update"
    global source
    source.stream(dict(x=, y=[y]), 345600)

p = figure(y_range=[0,1000], plot_width=1000)

p.line(x='x', y='y', source=source)
doc.add_root(p)

thread = Thread(target = notify)
thread.start()
bokeh serve --port 51

On Wednesday, 18 January 2017 10:05:16 UTC-5, Eugene Cheung wrote:
Hi everyone.

I have a graph I built using Bokeh that updates the graph (which contains multiple linegraphs) every time a notify message is sent from one of the Observable classes. The data point set (which includes a Key (which is the name of the line graph to update a timestamp and a value) is plot every time notify is called. for the first 2 hours the plot points are added seamlessly and the graph updates very quickly. But after that point the updates are trailing the notify message by a couple hours. (i.e. when i'm streaming time (14:50:18.518, val 89.000) its only graphing 10:51:18.532, val 76 - this is after 8 hours of streaming data in). I need a days worth of data to be shown which is why my stream buffer is 345600. The weird thing that confuses me is that the plot continue to update data from a couple hours ago, even though its a couple hours behind the real time inputs... my guess is that there is a data buffer that is pushing the values onto the graph.

Is there anything I am doing wrong? is there anyway to speed up the process? workarounds? Would down sampling possibly help? Thanks for your help!

Here is my code.
class LiveGrapher: #Observer
    
    def __init__(self, observable):
        self.datasetsDisplayed = observable.registerObserver(self)
        self.source = {}
        self.colors = ['red', 'green', 'blue', 'black', 'purple', 'brown','AliceBlue', 'AniqueWhite', 'Aqua', 'Aquamarine', 'Azure', 'Beige', 'Bisque']
        for item in self.datasetsDisplayed:
            self.source[item] = ColumnDataSource(data=dict(x=, y=, dateStampStr = , timeStampStr = , key = ))
            
        self.doc = curdoc()
    
    def notify(self, observable, *args, **kwargs):
        timeStamp = ''
        YKey = ''
        YVal = 0
        timeStamp = kwargs['timeStamp']
        for key in kwargs.keys():
            if key in self.datasetsDisplayed:
               YKey = key
               YVal = kwargs[YKey]
               self.doc.add_next_tick_callback(partial(self.update, x=timeStamp, y=YVal, key =YKey))
               
    def update(self, x, y, key):
        self.source[key].stream(dict(x=, y=[y], key = [key], dateStampStr=[x.strftime('%m/%d/%Y')], timeStampStr=[x.strftime('%H:%M:%S')]), 345600)

    def draw(self):
        drawTools = [BoxZoomTool(), ResetTool(), HoverTool( tooltips=[('key', '@key'), ('date', '@dateStampStr'), ('time', '@timeStampStr'), ('value' , '@y')])]
        
        p = figure(x_axis_type="datetime", y_range=[0,2000], tools=drawTools, plot_width=1000)
        
        colorID = 0
        for dataset in self.datasetsDisplayed:
            p.line(x='x', y='y', legend = dataset, source=self.source[dataset], line_color=self.colors[colorID])
            #p.square(x='x', y='y', legend= dataset, source=self.source[dataset], fill_color=None, line_color=self.colors[colorID])
            colorID = colorID + 1
        p.legend.location = "top_right"
        p.legend.background_fill_color = "grey"
        p.legend.background_fill_alpha = 0.5
        
        self.doc.add_root(p)

--
You received this message because you are subscribed to the Google Groups "Bokeh Discussion - Public" group.
To unsubscribe from this group and stop receiving emails from it, send an email to [email protected].
To post to this group, send email to [email protected].
To view this discussion on the web visit https://groups.google.com/a/continuum.io/d/msgid/bokeh/c7b0bf9d-b781-40a9-9b48-bbff03419f52%40continuum.io\.
For more options, visit https://groups.google.com/a/continuum.io/d/optout\.

Just tried enabling webgl, (p = figure(webgl=True, y_range=[0,1000], plot_width=1000)) but it for some reason causesmy figure to start shrinking about 10 secnonds into the applications runtime.

···

On Wednesday, 18 January 2017 13:16:51 UTC-5, Bryan Van de ven wrote:

It’s possible. There is a webgl option you could try. I believe it may currently have some regressions on safari, but for anything over a few tens of thousands of points, it’s probably worth considering.

Thanks,

Bryan

On Jan 18, 2017, at 11:34 AM, Eugene Cheung [email protected] wrote:

So i stripped down my code so that it updates the x position every 0.01 seconds and it appears that what the graph displays and my actual count starts to differ at around ~ 15000 mark (which was right around the point my code above started to slow down (71 minutes in, displaying about 15000 points). Is it a pure drawing issue?

YVal = 200

XVal = 0

source = ColumnDataSource(data=dict(x=, y=))

doc = curdoc()
def notify():

global YVal, XVal, doc
while True:
    print "i am here in notify"
    XVal = XVal + 1
    time.sleep(0.01)
    print XVal
    doc.add_next_tick_callback(partial(update, x=XVal, y=YVal))

def update( x, y):

print "i am here in update"
global source
source.stream(dict(x=[x], y=[y]), 345600)

p = figure(y_range=[0,1000], plot_width=1000)

p.line(x=‘x’, y=‘y’, source=source)

doc.add_root(p)

thread = Thread(target = notify)

thread.start()

bokeh serve --port 51

On Wednesday, 18 January 2017 10:05:16 UTC-5, Eugene Cheung wrote:

Hi everyone.

I have a graph I built using Bokeh that updates the graph (which contains multiple linegraphs) every time a notify message is sent from one of the Observable classes. The data point set (which includes a Key (which is the name of the line graph to update a timestamp and a value) is plot every time notify is called. for the first 2 hours the plot points are added seamlessly and the graph updates very quickly. But after that point the updates are trailing the notify message by a couple hours. (i.e. when i’m streaming time (14:50:18.518, val 89.000) its only graphing 10:51:18.532, val 76 - this is after 8 hours of streaming data in). I need a days worth of data to be shown which is why my stream buffer is 345600. The weird thing that confuses me is that the plot continue to update data from a couple hours ago, even though its a couple hours behind the real time inputs… my guess is that there is a data buffer that is pushing the values onto the graph.

Is there anything I am doing wrong? is there anyway to speed up the process? workarounds? Would down sampling possibly help? Thanks for your help!

Here is my code.

class LiveGrapher: #Observer

def __init__(self, observable):
    self.datasetsDisplayed = observable.registerObserver(self)
    self.source = {}
    self.colors = ['red', 'green', 'blue', 'black', 'purple', 'brown','AliceBlue', 'AniqueWhite', 'Aqua', 'Aquamarine', 'Azure', 'Beige', 'Bisque']
    for item in self.datasetsDisplayed:
        self.source[item] = ColumnDataSource(data=dict(x=[], y=[], dateStampStr = [], timeStampStr = [], key = []))
       
    self.doc = curdoc()  

def notify(self, observable, *args, **kwargs):
    timeStamp = ''
    YKey = ''
    YVal = 0
    timeStamp = kwargs['timeStamp']
    for key in kwargs.keys():
        if key in self.datasetsDisplayed:
           YKey = key
           YVal = kwargs[YKey]
           self.doc.add_next_tick_callback(partial(self.update, x=timeStamp, y=YVal, key =YKey))
def update(self, x, y, key):
    self.source[key].stream(dict(x=[x], y=[y], key = [key], dateStampStr=[x.strftime('%m/%d/%Y')], timeStampStr=[x.strftime('%H:%M:%S')]), 345600)
def draw(self):
    drawTools = [BoxZoomTool(), ResetTool(), HoverTool( tooltips=[('key', '@key'), ('date', '@dateStampStr'), ('time', '@timeStampStr'), ('value' , '@y')])]
    p = figure(x_axis_type="datetime",  y_range=[0,2000], tools=drawTools, plot_width=1000)
    colorID = 0
    for dataset in self.datasetsDisplayed:
        p.line(x='x', y='y', legend = dataset, source=self.source[dataset], line_color=self.colors[colorID])
        #p.square(x='x', y='y', legend= dataset, source=self.source[dataset], fill_color=None, line_color=self.colors[colorID])
        colorID = colorID + 1
    p.legend.location = "top_right"
    p.legend.background_fill_color = "grey"
    p.legend.background_fill_alpha = 0.5
    self.doc.add_root(p)


You received this message because you are subscribed to the Google Groups “Bokeh Discussion - Public” group.

To unsubscribe from this group and stop receiving emails from it, send an email to [email protected].

To post to this group, send email to [email protected].

To view this discussion on the web visit https://groups.google.com/a/continuum.io/d/msgid/bokeh/c7b0bf9d-b781-40a9-9b48-bbff03419f52%40continuum.io.

For more options, visit https://groups.google.com/a/continuum.io/d/optout.

That's unfortunate. The person who originally wrote the webgl code and is the person who knows about webgl is not contributing to the project regularly anymore, so I can't promise any quick fixes around webgl until someone with the right experience decides to become a contributor.

Given that, Bokeh may not be the right option for you if you need to send all that data, all the time. ~350k points is just too much for the canvas backend. If you don't really need all those points in the browser, some options are:

* downsample yourself, in some way, based on the current viewable range
* use Datashader to help to the same

Bryan

···

On Jan 18, 2017, at 1:44 PM, Eugene Cheung <[email protected]> wrote:

Just tried enabling webgl, (p = figure(webgl=True, y_range=[0,1000], plot_width=1000)) but it for some reason causesmy figure to start shrinking about 10 secnonds into the applications runtime.

On Wednesday, 18 January 2017 13:16:51 UTC-5, Bryan Van de ven wrote:
It's possible. There is a webgl option you could try. I believe it may currently have some regressions on safari, but for anything over a few tens of thousands of points, it's probably worth considering.

Thanks,

Bryan

> On Jan 18, 2017, at 11:34 AM, Eugene Cheung <[email protected]> wrote:
>
> So i stripped down my code so that it updates the x position every 0.01 seconds and it appears that what the graph displays and my actual count starts to differ at around ~ 15000 mark (which was right around the point my code above started to slow down (71 minutes in, displaying about 15000 points). Is it a pure drawing issue?
> YVal = 200
> XVal = 0
> source = ColumnDataSource(data=dict(x=, y=))
> doc = curdoc()
> def notify():
> global YVal, XVal, doc
> while True:
> print "i am here in notify"
> XVal = XVal + 1
> time.sleep(0.01)
> print XVal
> doc.add_next_tick_callback(partial(update, x=XVal, y=YVal))
> def update( x, y):
> print "i am here in update"
> global source
> source.stream(dict(x=, y=[y]), 345600)
>
>
> p = figure(y_range=[0,1000], plot_width=1000)
>
> p.line(x='x', y='y', source=source)
> doc.add_root(p)
>
> thread = Thread(target = notify)
> thread.start()
> bokeh serve --port 51
>
> On Wednesday, 18 January 2017 10:05:16 UTC-5, Eugene Cheung wrote:
> Hi everyone.
>
> I have a graph I built using Bokeh that updates the graph (which contains multiple linegraphs) every time a notify message is sent from one of the Observable classes. The data point set (which includes a Key (which is the name of the line graph to update a timestamp and a value) is plot every time notify is called. for the first 2 hours the plot points are added seamlessly and the graph updates very quickly. But after that point the updates are trailing the notify message by a couple hours. (i.e. when i'm streaming time (14:50:18.518, val 89.000) its only graphing 10:51:18.532, val 76 - this is after 8 hours of streaming data in). I need a days worth of data to be shown which is why my stream buffer is 345600. The weird thing that confuses me is that the plot continue to update data from a couple hours ago, even though its a couple hours behind the real time inputs... my guess is that there is a data buffer that is pushing the values onto the graph.
>
> Is there anything I am doing wrong? is there anyway to speed up the process? workarounds? Would down sampling possibly help? Thanks for your help!
>
> Here is my code.
> class LiveGrapher: #Observer
>
> def __init__(self, observable):
> self.datasetsDisplayed = observable.registerObserver(self)
> self.source = {}
> self.colors = ['red', 'green', 'blue', 'black', 'purple', 'brown','AliceBlue', 'AniqueWhite', 'Aqua', 'Aquamarine', 'Azure', 'Beige', 'Bisque']
> for item in self.datasetsDisplayed:
> self.source[item] = ColumnDataSource(data=dict(x=, y=, dateStampStr = , timeStampStr = , key = ))
>
> self.doc = curdoc()
>
> def notify(self, observable, *args, **kwargs):
> timeStamp = ''
> YKey = ''
> YVal = 0
> timeStamp = kwargs['timeStamp']
> for key in kwargs.keys():
> if key in self.datasetsDisplayed:
> YKey = key
> YVal = kwargs[YKey]
> self.doc.add_next_tick_callback(partial(self.update, x=timeStamp, y=YVal, key =YKey))
>
>
>
> def update(self, x, y, key):
> self.source[key].stream(dict(x=, y=[y], key = [key], dateStampStr=[x.strftime('%m/%d/%Y')], timeStampStr=[x.strftime('%H:%M:%S')]), 345600)
>
> def draw(self):
> drawTools = [BoxZoomTool(), ResetTool(), HoverTool( tooltips=[('key', '@key'), ('date', '@dateStampStr'), ('time', '@timeStampStr'), ('value' , '@y')])]
>
> p = figure(x_axis_type="datetime", y_range=[0,2000], tools=drawTools, plot_width=1000)
>
> colorID = 0
> for dataset in self.datasetsDisplayed:
> p.line(x='x', y='y', legend = dataset, source=self.source[dataset], line_color=self.colors[colorID])
> #p.square(x='x', y='y', legend= dataset, source=self.source[dataset], fill_color=None, line_color=self.colors[colorID])
> colorID = colorID + 1
> p.legend.location = "top_right"
> p.legend.background_fill_color = "grey"
> p.legend.background_fill_alpha = 0.5
>
>
> self.doc.add_root(p)
>
> --
> You received this message because you are subscribed to the Google Groups "Bokeh Discussion - Public" group.
> To unsubscribe from this group and stop receiving emails from it, send an email to bokeh+un...@continuum.io.
> To post to this group, send email to bo...@continuum.io.
> To view this discussion on the web visit https://groups.google.com/a/continuum.io/d/msgid/bokeh/c7b0bf9d-b781-40a9-9b48-bbff03419f52%40continuum.io\.
> For more options, visit https://groups.google.com/a/continuum.io/d/optout\.

--
You received this message because you are subscribed to the Google Groups "Bokeh Discussion - Public" group.
To unsubscribe from this group and stop receiving emails from it, send an email to [email protected].
To post to this group, send email to [email protected].
To view this discussion on the web visit https://groups.google.com/a/continuum.io/d/msgid/bokeh/157bc4d9-f67d-430a-867d-4e44b3cff2b2%40continuum.io\.
For more options, visit https://groups.google.com/a/continuum.io/d/optout\.