Use @property for properties
[joel/kofoto.git] / src / packages / kofoto / gkofoto / cachingpixbufloader.py
1 """This module contains the CachingPixbufLoader class."""
2
3 __all__ = ["CachingPixbufLoader"]
4
5 import gc
6 import os
7 import time
8 if __name__ == "__main__":
9     import pygtk
10     pygtk.require("2.0")
11 import gobject
12 from kofoto.gkofoto.pixbufloader import PixbufLoader, get_pixbuf_size
13 from kofoto.gkofoto.pseudothread import PseudoThread
14 from kofoto.iodict import InsertionOrderedDict
15 from kofoto.rectangle import Rectangle
16
17 class _RequestStateBase(object):
18     def __init__(self, request):
19         self._request = request
20
21     def add_callback(self, load_callback, error_callback):
22         self._request._callbacks.append((load_callback, error_callback))
23
24     def get_number_of_loaded_pixels(self):
25         if self._request._loaded_bytes == 0:
26             return 0
27         else:
28             return (
29                 float(self._request._size_to_load[0]) *
30                 self._request._size_to_load[1] *
31                 self._request._loaded_bytes /
32                 self._request._available_bytes)
33
34     def is_finished(self):
35         raise NotImplementedError
36
37     def load_some_more(self):
38         return 0
39
40 class _RequestStateInitial(_RequestStateBase):
41     def get_number_of_loaded_pixels(self):
42         return 0
43
44     def is_finished(self):
45         return False
46
47     def load_some_more(self):
48         req = self._request
49         try:
50             stat = os.stat(req._path)
51             # TODO: Look for image in diskcache here and load the
52             # cached image instead (if cached image mtime <= original
53             # image mtime?).
54         except OSError:
55             req._state = _RequestStateError(req)
56         else:
57             req._mtime = stat.st_mtime
58             req._available_bytes = stat.st_size
59             req._state = _RequestStateWaitingForSize(req)
60             req._pb_loader.prepare(req._path, req._size_limit)
61         return 0
62
63 class _RequestStateWaitingForSize(_RequestStateBase):
64     def get_number_of_loaded_pixels(self):
65         return 0
66
67     def is_finished(self):
68         return False
69
70     def load_some_more(self):
71         req = self._request
72         loaded_bytes = req._pb_loader.load_some_more()
73         req._unreported_bytes += loaded_bytes
74         original_size = req._pb_loader.get_original_size()
75         if original_size is not None:
76             req._state = _RequestStateLoading(req)
77             req._original_size = original_size
78             if req._size_limit is None:
79                 req._size_to_load = original_size
80             else:
81                 req._size_to_load = Rectangle(*original_size).downscaled_to(
82                     req._size_limit)
83         elif loaded_bytes == 0:
84             # Could not parse image size.
85             req._state = _RequestStateError(req)
86         return 0
87
88 class _RequestStateLoading(_RequestStateBase):
89     def is_finished(self):
90         return False
91
92     def load_some_more(self):
93         req = self._request
94         loaded_bytes = req._pb_loader.load_some_more()
95         if req._unreported_bytes > 0:
96             loaded_bytes += req._unreported_bytes
97             req._unreported_bytes = 0
98         req._loaded_bytes += loaded_bytes
99         if loaded_bytes == 0:
100             # Here it would be possible to check mtime again and
101             # reload the image if mtime has changed. We decided to
102             # ignore this right now because of implementation
103             # difficulties (for example, a negative number of pixels
104             # (watch out so it isn't 0!) must be reported back to
105             # req._cpb_loader in some way).
106
107             req._pixbuf = req._pb_loader.get_pixbuf()
108             if req._pixbuf is None:
109                 req._state = _RequestStateError(req)
110             else:
111                 req._state = _RequestStateFinished(req)
112         return (
113             (float(loaded_bytes) / req._available_bytes) *
114             req._size_to_load[0] * req._size_to_load[1])
115
116 class _RequestStateFinished(_RequestStateBase):
117     def __init__(self, request):
118         _RequestStateBase.__init__(self, request)
119         for (load_cb, _) in request._callbacks:
120             if request._cpb_loader._debug_level > 0:
121                 print "%.3f callback(%s, %s)" % (
122                     time.time(), request._path, request._size_limit)
123             load_cb(request._pixbuf, request._original_size)
124         request._callbacks = []
125         # TODO: Store in disk cache here if
126         # self._request._persistence_size_limit isn't None and we
127         # didn't load the image from the cache.
128
129     def add_callback(self, load_callback, _):
130         req = self._request
131         if req._cpb_loader._debug_level > 0:
132             print "%.3f callback(%s, %s)" % (
133                 time.time(), req._path, req._size_limit)
134         load_callback(req._pixbuf, req._original_size)
135
136     def is_finished(self):
137         return True
138
139 class _RequestStateError(_RequestStateBase):
140     def __init__(self, request):
141         _RequestStateBase.__init__(self, request)
142         request._pb_loader.cancel()
143         request._pixbuf = None
144         for (_, error_cb) in request._callbacks:
145             if error_cb is not None:
146                 error_cb()
147         request._callbacks = []
148
149         # Don't cache negative requests since we can't be sure that it
150         # will fail in the future. The image may for example be partly
151         # written now but fully written soon.
152         key = (request._path, request._size_limit)
153         request._cpb_loader._remove_erroneous_request(key)
154
155     def add_callback(self, _, error_callback):
156         error_callback()
157
158     def is_finished(self):
159         return True
160
161 class _Request(object):
162     def __init__(self, cpb_loader, path, size_limit, persistence_size_limit):
163         self._cpb_loader = cpb_loader
164         self._path = path
165         self._size_limit = size_limit
166         self._persistence_size_limit = persistence_size_limit
167         self._callbacks = [] # List of (load_callback, error_callback).
168         self._pixbuf = None
169         self._unreported_bytes = 0
170         self._loaded_bytes = 0
171         self._available_bytes = None
172         self._original_size = None
173         self._size_to_load = None
174         self._mtime = None
175         self._pb_loader = PixbufLoader()
176         self._state = _RequestStateInitial(self)
177
178     # ----------------------------------
179
180     @property
181     def key(self):
182         return (self._path, self._size_limit)
183
184     # ----------------------------------
185
186     def add_callback(self, load_callback, error_callback):
187         return self._state.add_callback(load_callback, error_callback)
188
189     def cancel(self):
190         self._pb_loader.cancel()
191         del self._state # Break cycle.
192
193     def get_number_of_loaded_pixels(self):
194         return self._state.get_number_of_loaded_pixels()
195
196     def has_changed_on_disk(self):
197         if self._mtime is None:
198             # We don't know yet.
199             return False
200         else:
201             try:
202                 return os.path.getmtime(self._path) != self._mtime
203             except OSError:
204                 return True
205
206     def is_finished(self):
207         return self._state.is_finished()
208
209     def load_some_more(self):
210         return self._state.load_some_more()
211
212     def remove_callback(self, load_callback, error_callback):
213         try:
214             self._callbacks.remove((load_callback, error_callback))
215         except ValueError:
216             pass
217
218     def _print_state(self, verbose):
219         print "        state:                 ", self._state.__class__.__name__
220         print "        path:                  ", self._path
221         print "        size limit:            ", self._size_limit
222         print "        persistence size limit:", self._persistence_size_limit
223         if verbose:
224             print "        callbacks:             ", self._callbacks
225             print "        pixbuf:                ", self._pixbuf
226             print "        unreported bytes:      ", self._unreported_bytes
227             print "        loaded bytes:          ", self._loaded_bytes
228             print "        available bytes:       ", self._available_bytes
229             print "        original size:         ", self._original_size
230             print "        size to load:          ", self._size_to_load
231             print "        mtime:                 ", self._mtime
232             print "        pb loader:             ", self._pb_loader
233
234 class CachingPixbufLoader(object):
235     """A pixbuf loader with preload and cache functionality.
236
237     This class is a pixbuf loader that keeps loaded pixbufs in an LRU
238     memory cache. It handles several outstanding pixbuf load requests
239     and also handles preload requests. It can optionally store pixbufs
240     in a disk-based cache. Currently, all pixbufs are stored as JPEGs.
241
242     Load requests are asynchronous; the request is done by supplying
243     the path to the image, a wanted pixbuf size and a callback
244     function to the load() method. The callback function is called
245     when the load has finished (or immediately, if the pixbuf already
246     exists in the cache). Old load requests have higher priority than
247     new. Load requests have higher priority than preload requests.
248     Load requests are never forgotten.
249
250     Preload requests are also asynchronous. A preload request is used
251     to hint the loader to load a pixbuf into the cache. New preload
252     requests have higher priority than old. Load requests have higher
253     priority than preload requests. Old preload requests may be
254     ignored (and the cached pixbuf thrown away) if the cache limits
255     are exceeded.
256     """
257
258     def __init__(self, pixel_limit=10**7, cache_directory=None):
259         """Constructor.
260
261         Arguments:
262
263         pixel_limit  -- The number of pixels to keep in the memory
264                         cache.
265         cache_directory
266                      -- Path to the disk cache directory. If None, no
267                         images will be cached on disk.
268         """
269
270         self._pixel_limit = pixel_limit
271         self._cache_directory = cache_directory
272         self._load_thread = PseudoThread(self._load_loop())
273         self._debug_level = 0
274
275         # Cached value of the sum of loaded pixels of all requests in
276         # the queue.
277         self._pixels_in_cache = 0.0
278
279         # Whether the load loop should reevaluate which request to
280         # work on.
281         self._load_loop_reeval = False
282
283         # Maps path to Rectangle(full_width, full_height).
284         self._available_size = {}
285
286         # Maps (path, (width_limit, height_limit)) to _Request
287         # instances. Newest requests are first and oldest are last.
288         self._request_queue = InsertionOrderedDict()
289
290         # List of _Request instances. Always a subset of
291         # self._request_queue.values() (except order). Newest requests
292         # are last and oldest are first.
293         self._load_queue = []
294
295         # Maps path to set(_Request)
296         self._path_to_requests = {}
297
298     def cancel_load(self, handle):
299         """Cancel a load.
300
301         This method does not remove the cached pixbuf from the cache,
302         it just makes sure that the callback passed to load doesn't
303         get called. In other words, cancel_load converts a load to a
304         preload.
305
306         If the handle represents a load that already has finished,
307         nothing will happen and no exception will be raised.
308         """
309         if self._debug_level > 0:
310             print "%.3f cancel_load(%r)" % (time.time(), handle)
311
312         (path, size_limit, load_callback, error_callback) = handle
313         key = (path, size_limit)
314         if key in self._request_queue:
315             request = self._request_queue[key]
316             try:
317                 self._load_queue.remove(request)
318             except ValueError:
319                 # Nothing to do.
320                 return
321             request.remove_callback(load_callback, error_callback)
322             self._load_loop_reeval = True
323
324     def get_pixel_limit(self):
325         """Get cache size limit."""
326
327         return self._pixel_limit
328
329     def load(self, path, size_limit, load_callback,
330              error_callback=None, persistence_size_limit=None):
331         """Load a pixbuf as quick as possible.
332
333         The pixbuf is taken from the cache if possible.
334
335         Old load requests have higher priority than new. Load requests
336         have higher priority than preload requests.
337
338         Arguments:
339
340         path         -- Path to the image file to load.
341         size_limit   -- A tuple (width, height) with the size limit of
342                         the resulting pixbuf, or None. If None, a
343                         full-size pixbuf will be loaded.
344         load_callback
345                      -- Function to call when loading has finished.
346                         The function will be passed two arguments: the
347                         resulting pixbuf and a tuple (width, height)
348                         with the full size of the image file on disk.
349         error_callback
350                      -- Function to call when loading has failed. If
351                         None, no call will be made and the error will
352                         be ignored. The function will be given no
353                         arguments.
354         persistence_size_limit
355                      -- Limit (an integer) of the image stored in the
356                         disk cache (if disk caching is enabled). Must
357                         be None if size_limit is None. Must be equal
358                         to or greater than max(size_limit[0],
359                         size_limit[1]). If None, the disk cache will
360                         not be used.
361
362         The method returns a handle that can be passed to cancel_load.
363         """
364         if self._debug_level > 0:
365             print "%.3f load(%s, %s)" % (time.time(), path, size_limit)
366
367         if size_limit is not None:
368             size_limit = tuple(size_limit)
369             if persistence_size_limit is not None:
370                 assert persistence_size_limit >= size_limit[0]
371                 assert persistence_size_limit >= size_limit[1]
372             size_limit = self._calculate_size_limit(path, size_limit)
373         key = (path, size_limit)
374         request = self._request_queue.get(key)
375         if request and request.has_changed_on_disk():
376             self._remove_request(key)
377         if key in self._request_queue:
378             # Just move it to the front.
379             request = self._request_queue[key]
380             self._request_queue.insert_first(key, request)
381         else:
382             # Let preload create the request.
383             self.preload(path, size_limit, persistence_size_limit)
384             request = self._request_queue[key]
385         if request not in self._load_queue:
386             self._load_queue.append(request)
387             self._load_loop_reeval = True
388
389         # It's okay to wait until now to add the callback, since any
390         # errors that will trigger error_callback will arise later (at
391         # the first load_some_more in the load loop).
392         request.add_callback(load_callback, error_callback)
393
394         self._load_thread.start()
395         handle = (
396             path, size_limit, load_callback, error_callback)
397         return handle
398
399     def preload(self, path, size_limit, persistence_size_limit=None):
400         """Request that a pixbuf should be loaded into the cache.
401
402         The pixbuf can be retrieved by a call to load().
403
404         New preload requests have higher priority than old. Load
405         requests have higher priority than preload requests.
406
407         Arguments:
408
409         path         -- Path to the image file to load.
410         size_limit   -- A tuple (width, height) with the size limit of the
411                         resulting pixbuf, or None. If None, a
412                         full-size pixbuf will be loaded.
413         persistence_size_limit
414                      -- Limit (an integer) of the image stored in the
415                         disk cache (if disk caching is enabled). Must
416                         be None if size_limit is None. Must be equal
417                         to or greater than max(size_limit[0],
418                         size_limit[1]). If None, the disk cache will
419                         not be used.
420         """
421
422         if self._debug_level > 0:
423             print "%.3f preload(%s, %s)" % (time.time(), path, size_limit)
424
425         if size_limit is not None:
426             size_limit = tuple(size_limit)
427             if persistence_size_limit is not None:
428                 assert persistence_size_limit >= size_limit[0]
429                 assert persistence_size_limit >= size_limit[1]
430             size_limit = self._calculate_size_limit(path, size_limit)
431         key = (path, size_limit)
432         if key in self._request_queue:
433             request = self._request_queue[key]
434         else:
435             request = _Request(self, path, size_limit, persistence_size_limit)
436             request_set = self._path_to_requests.setdefault(path, set())
437             request_set.add(request)
438         self._request_queue.insert_first(key, request)
439         self._load_loop_reeval = True
440         self._load_thread.start()
441
442     def set_pixel_limit(self, pixel_limit):
443         """Set cache size limit.
444
445         Arguments:
446
447         pixel_limit -- The number of pixels to keep in the cache.
448         """
449
450         self._pixel_limit = pixel_limit
451         self._prune_queue()
452
453     def unload(self, path, size_limit):
454         """Remove a cached pixbuf from the cache.
455
456         This method is a hint to the cache that it's not interesting
457         to keep a cached pixbuf anymore. If no pixbuf for the given
458         path and size limit exists, nothing will happen.
459
460         Arguments:
461
462         path         -- Path given to load/preload.
463         size_limit   -- Size limit given to load/preload.
464         """
465
466         if self._debug_level > 0:
467             print "%.3f unload(%s, %s)" % (time.time(), path, size_limit)
468
469         key = (path, size_limit)
470         if key in self._request_queue:
471             request = self._request_queue[key]
472             if request not in self._load_queue:
473                 self._remove_request(key)
474
475     def unload_all(self, path):
476         """Remove all cached sizes of a pixbuf from the cache.
477
478         This method is a hint to the cache that it's not interesting
479         to keep a cached pixbuf anymore. If no pixbufs for the given
480         path exist, nothing will happen.
481
482         Arguments:
483
484         path         -- Path given to load/preload.
485         """
486
487         if self._debug_level > 0:
488             print "%.3f unload_all(%s)" % (time.time(), path)
489
490         if path not in self._path_to_requests:
491             return
492         for request in self._path_to_requests[path].copy():
493             if request not in self._load_queue:
494                 self._remove_request(request.key)
495
496     def _calculate_size_limit(self, path, size_limit):
497         if size_limit is None:
498             return None
499         else:
500             available_size = self._available_size.get(path)
501             if available_size is None:
502                 size = get_pixbuf_size(path)
503                 if size is None:
504                     # Error reading image size.
505                     return None
506                 available_size = Rectangle(*size)
507                 self._available_size[path] = available_size
508             if available_size.fits_within(size_limit):
509                 return None
510             else:
511                 return size_limit
512
513     def _load_loop(self):
514         while True:
515             request = None
516             self._load_loop_reeval = False
517             found_a_load = False
518             while len(self._load_queue) > 0:
519                 if self._load_queue[0].is_finished():
520                     # The load request has finished, so remove it.
521                     del self._load_queue[0]
522                 else:
523                     request = self._load_queue[0]
524                     found_a_load = True
525                     break
526             if request is None:
527                 # No loads. Find a preload, if any.
528                 for req in self._request_queue.itervalues():
529                     if not req.is_finished():
530                         request = req
531                         break
532             if request is None:
533                 # The request queue needs to be pruned now since there
534                 # may be old load requests that now are finished but
535                 # could not be removed before.
536                 self._prune_queue()
537
538                 # No loads or preloads left. Pause until
539                 # self._load_thread.start() is called.
540                 if self._debug_level > 1:
541                     print
542                     print "LOAD LOOP FINISHED:"
543                     self._print_state(False)
544                 self._load_thread.stop()
545                 yield True
546             else:
547                 if self._debug_level > 1:
548                     print "LOAD LOOP SELECTED TO LOAD", request._path
549                 if found_a_load:
550                     priority = gobject.PRIORITY_HIGH_IDLE
551                 else:
552                     priority = gobject.PRIORITY_DEFAULT_IDLE
553                 self._load_thread.set_priority(priority)
554                 while not (self._load_loop_reeval or request.is_finished()):
555                     loaded_pixels = request.load_some_more()
556                     self._pixels_in_cache += loaded_pixels
557                     self._prune_queue()
558                     yield True
559             # Now it's time to reevaluate which request to work on.
560
561     def _print_state(self, verbose=False):
562         print "------------------------------------------------------------------"
563         print "Pixel limit:", self._pixel_limit
564         print "Cache directory:", self._cache_directory
565         print "Pixels in cache:", self._pixels_in_cache
566         print "Request queue:"
567         for (key, request) in self._request_queue.iteritems():
568             print "    Key:", key
569             print "    Request:"
570             request._print_state(verbose)
571         print "Load queue:"
572         for request in self._load_queue:
573             print "    Request:"
574             request._print_state(verbose)
575         print "Path to requests:"
576         for (path, requests) in self._path_to_requests.iteritems():
577             print "    Path:", path
578             print "    Requests:", requests
579
580     def _prune_queue(self):
581         if self._pixels_in_cache <= self._pixel_limit:
582             return
583         requests_to_prune = []
584         load_requests = set(self._load_queue)
585         pixels_after_pruning = self._pixels_in_cache
586         for (key, request) in self._request_queue.reviteritems():
587             if pixels_after_pruning <= self._pixel_limit:
588                 # Enough pruning.
589                 break
590             if request not in load_requests:
591                 requests_to_prune.append((key, request))
592                 pixels_after_pruning -= request.get_number_of_loaded_pixels()
593         if len(requests_to_prune) > 0:
594             self._load_loop_reeval = True
595             for (key, request) in requests_to_prune:
596                 if self._debug_level > 1:
597                     print "PRUNING", key
598                 self._remove_request(key)
599             gc.collect()
600
601     def _remove_erroneous_request(self, key):
602         self._remove_request(key)
603
604     def _remove_request(self, key):
605         assert key in self._request_queue
606         request = self._request_queue[key]
607         self._pixels_in_cache -= request.get_number_of_loaded_pixels()
608         request.cancel()
609         try:
610             self._load_queue.remove(request)
611         except ValueError:
612             # OK, it wasn't a load.
613             pass
614         del self._request_queue[key]
615         path = key[0]
616         request_set = self._path_to_requests[path]
617         request_set.remove(request)
618         if len(request_set) == 0:
619             del self._path_to_requests[path]
620
621 ######################################################################
622
623 def main(argv):
624     import gtk
625
626     def pixbuf_loaded_cb(pixbuf, full_size):
627         print "Received pixbuf of %dx%d pixels (full size: %dx%d)." % (
628             pixbuf.get_width(), pixbuf.get_height(), full_size[0], full_size[1])
629     def pixbuf_error_cb():
630         print "Error while loading pixbuf."
631
632     loader = CachingPixbufLoader()
633     loader._debug_level = 2
634     loader.set_pixel_limit(10000000000)
635     print "INITIAL:"
636     loader._print_state(False)
637
638     for path in argv[1:]:
639         loader.load(path, (300, 200), pixbuf_loaded_cb, pixbuf_error_cb)
640
641     print
642     print "BEFORE LOAD LOOP START:"
643     loader._print_state(False)
644
645     gtk.main()
646
647 if __name__ == "__main__":
648     import sys
649     main(sys.argv)