Package flumotion :: Package component :: Package misc :: Package httpserver :: Module cachestats
[hide private]

Source Code for Module flumotion.component.misc.httpserver.cachestats

  1  # -*- Mode: Python; test-case-name: -*- 
  2  # vi:si:et:sw=4:sts=4:ts=4 
  3   
  4  # Flumotion - a streaming media server 
  5  # Copyright (C) 2004,2005,2006,2007,2008,2009 Fluendo, S.L. 
  6  # Copyright (C) 2010,2011 Flumotion Services, S.A. 
  7  # All rights reserved. 
  8  # 
  9  # This file may be distributed and/or modified under the terms of 
 10  # the GNU Lesser General Public License version 2.1 as published by 
 11  # the Free Software Foundation. 
 12  # This file is distributed without any warranty; without even the implied 
 13  # warranty of merchantability or fitness for a particular purpose. 
 14  # See "LICENSE.LGPL" in the source distribution for more information. 
 15  # 
 16  # Headers in this file shall remain intact. 
 17   
 18  import time 
 19   
 20  from twisted.internet import reactor 
 21   
 22  from flumotion.common import log 
 23   
 24   
 25  # Filter out requests that read less than a block for average values 
 26  MIN_REQUEST_SIZE = 64 * 1024 + 1 
 27  # Statistics update period 
 28  STATS_UPDATE_PERIOD = 10 
 29   
 30  CACHE_MISS = 0 
 31  CACHE_HIT = 1 
 32  TEMP_HIT = 2 
 33   
 34   
35 -class RequestStatistics(object):
36
37 - def __init__(self, cacheStats):
38 self._stats = cacheStats 39 self._outdated = False 40 self._size = 0L 41 self._status = None 42 self.bytesReadFromSource = 0L 43 self.bytesReadFromCache = 0L
44
45 - def getBytesRead(self):
46 return self.bytesReadFromSource + self.bytesReadFromCache
47 bytesRead = property(getBytesRead) 48
49 - def getCacheReadRatio(self):
50 total = self.bytesRead 51 if total == 0: 52 return 0.0 53 return float(self.bytesReadFromCache) / total
54 cacheReadRatio = property(getCacheReadRatio) 55
56 - def onStarted(self, size, cacheStatus):
57 cs = self._stats 58 self._size = size 59 if cacheStatus == CACHE_HIT: 60 self._status = "cache-hit" 61 cs.cacheHitCount += 1 62 elif cacheStatus == TEMP_HIT: 63 cs.cacheHitCount += 1 64 cs.tempHitCount += 1 65 self._status = "temp-hit" 66 elif cacheStatus == CACHE_MISS: 67 cs.cacheMissCount += 1 68 if self._outdated: 69 self._status = "cache-outdate" 70 else: 71 self._status = "cache-miss" 72 cs._set("cache-hit-count", cs.cacheHitCount) 73 cs._set("temp-hit-count", cs.tempHitCount) 74 cs._set("cache-miss-count", cs.cacheMissCount)
75
76 - def onCacheOutdated(self):
77 self._outdated = True 78 cs = self._stats 79 cs.cacheOutdateCount += 1 80 cs._set("cache-outdate-count", cs.cacheOutdateCount)
81
82 - def onBytesRead(self, fromSource, fromCache, correction):
83 cs = self._stats 84 self.bytesReadFromSource += fromSource + correction 85 self.bytesReadFromCache += fromCache - correction 86 cs.bytesReadFromSource += fromSource + correction 87 cs.bytesReadFromCache += fromCache - correction
88
89 - def onClosed(self):
90 pass
91
92 - def getLogFields(self):
93 """ 94 Provide the following log fields: 95 cache-status: value can be 'cache-miss', 'cache-outdate', 96 'cache-hit', or 'temp-hit' 97 cache-read: how many bytes where read from the cache for 98 this resource. the difference from resource-read 99 was read from the source file (network file system?) 100 101 The proportion read from cache and from source are adjusted 102 to take into account the file copy. It's done by remembering 103 how many bytes are copied at session level. 104 """ 105 return {"cache-status": self._status, 106 "cache-read": self.bytesReadFromCache}
107 108
109 -class CacheStatistics(object):
110 111 _updater = None 112 _callId = None 113
114 - def __init__(self):
115 # For cache usage 116 self._cacheUsage = 0 117 self._cacheUsageRatio = 0.0 118 # For cache statistics 119 self.cacheHitCount = 0 120 self.tempHitCount = 0 121 self.cacheMissCount = 0 122 self.cacheOutdateCount = 0 123 self.cleanupCount = 0 124 # For real file reading statistics 125 self.bytesReadFromSource = 0L 126 self.bytesReadFromCache = 0L 127 # File copying statistics 128 self.totalCopyCount = 0 129 self.currentCopyCount = 0 130 self.finishedCopyCount = 0 131 self.cancelledCopyCount = 0 132 self.bytesCopied = 0L 133 self._copyRatios = 0.0
134
135 - def startUpdates(self, updater):
136 self._updater = updater 137 if updater and (self._callId is None): 138 self._set("cache-usage-estimation", self._cacheUsage) 139 self._set("cache-usage-ratio-estimation", self._cacheUsageRatio) 140 self._set("cleanup-count", self.cleanupCount) 141 self._set("last-cleanup-time", time.time()) 142 self._set("current-copy-count", self.currentCopyCount) 143 self._set("finished-copy-count", self.finishedCopyCount) 144 self._set("cancelled-copy-count", self.cancelledCopyCount) 145 self._set("mean-copy-ratio", self.meanCopyRatio) 146 self._set("mean-bytes-copied", self.meanBytesCopied) 147 self._update()
148
149 - def stopUpdates(self):
150 self._updater = None 151 if self._callId is not None: 152 self._callId.cancel() 153 self._callId = None
154
155 - def getCacheReadRatio(self):
156 total = self.bytesReadFromSource + self.bytesReadFromCache 157 if total == 0: 158 return 0 159 return float(self.bytesReadFromCache) / total
160 cacheReadRatio = property(getCacheReadRatio) 161
162 - def getMeanBytesCopied(self):
163 if self.finishedCopyCount == 0: 164 return 0 165 return self.bytesCopied / self.finishedCopyCount
166 meanBytesCopied = property(getMeanBytesCopied) 167
168 - def getMeanCopyRatio(self):
169 if self.finishedCopyCount == 0: 170 return 0 171 return self._copyRatios / self.finishedCopyCount
172 meanCopyRatio = property(getMeanCopyRatio) 173
174 - def onEstimateCacheUsage(self, usage, max):
175 self._cacheUsage = usage 176 self._cacheUsageRatio = float(usage) / max 177 self._set("cache-usage-estimation", self._cacheUsage) 178 self._set("cache-usage-ratio-estimation", self._cacheUsageRatio)
179
180 - def onCleanup(self):
181 self.cleanupCount += 1 182 self._set("cleanup-count", self.cleanupCount) 183 self._set("last-cleanup-time", time.time())
184
185 - def onCopyStarted(self):
186 self.currentCopyCount += 1 187 self.totalCopyCount += 1 188 self._set("current-copy-count", self.currentCopyCount)
189
190 - def onCopyCancelled(self, size, copied):
191 self.currentCopyCount -= 1 192 self.finishedCopyCount += 1 193 self.cancelledCopyCount += 1 194 self.bytesCopied += copied 195 self._copyRatios += float(copied) / size 196 self._set("current-copy-count", self.currentCopyCount) 197 self._set("finished-copy-count", self.finishedCopyCount) 198 self._set("cancelled-copy-count", self.cancelledCopyCount) 199 self._set("mean-copy-ratio", self.meanCopyRatio) 200 self._set("mean-bytes-copied", self.meanBytesCopied)
201
202 - def onCopyFinished(self, size):
203 self.currentCopyCount -= 1 204 self.finishedCopyCount += 1 205 self.bytesCopied += size 206 self._copyRatios += 1.0 207 self._set("current-copy-count", self.currentCopyCount) 208 self._set("finished-copy-count", self.finishedCopyCount) 209 self._set("mean-copy-ratio", self.meanCopyRatio) 210 self._set("mean-bytes-copied", self.meanBytesCopied)
211
212 - def _set(self, key, value):
213 if self._updater is not None: 214 self._updater.update(key, value)
215
216 - def _update(self):
217 self._set("cache-read-ratio", self.cacheReadRatio) 218 self._logStatsLine() 219 self._callId = reactor.callLater(STATS_UPDATE_PERIOD, self._update)
220
221 - def _logStatsLine(self):
222 """ 223 Statistic fields names: 224 CRR: Cache Read Ratio 225 CMC: Cache Miss Count 226 CHC: Cache Hit Count 227 THC: Temp Hit Count 228 COC: Cache Outdate Count 229 CCC: Cache Cleanup Count 230 CCU: Cache Current Usage 231 CUR: Cache Usage Ratio 232 PTC: coPy Total Count 233 PCC: coPy Current Count 234 PAC: coPy cAncellation Count 235 MCS: Mean Copy Size 236 MCR: Mean Copy Ratio 237 """ 238 log.debug("stats-local-cache", 239 "CRR: %.4f; CMC: %d; CHC: %d; THC: %d; COC: %d; " 240 "CCC: %d; CCU: %d; CUR: %.5f; " 241 "PTC: %d; PCC: %d; PAC: %d; MCS: %d; MCR: %.4f", 242 self.cacheReadRatio, self.cacheMissCount, 243 self.cacheHitCount, self.tempHitCount, 244 self.cacheOutdateCount, self.cleanupCount, 245 self._cacheUsage, self._cacheUsageRatio, 246 self.totalCopyCount, self.currentCopyCount, 247 self.cancelledCopyCount, self.meanBytesCopied, 248 self.meanCopyRatio)
249