1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 import time
19
20 from twisted.internet import reactor
21
22 from flumotion.common import log
23
24
25
26 MIN_REQUEST_SIZE = 64 * 1024 + 1
27
28 STATS_UPDATE_PERIOD = 10
29
30 CACHE_MISS = 0
31 CACHE_HIT = 1
32 TEMP_HIT = 2
33
34
36
38 self._stats = cacheStats
39 self._outdated = False
40 self._size = 0L
41 self._status = None
42 self.bytesReadFromSource = 0L
43 self.bytesReadFromCache = 0L
44
46 return self.bytesReadFromSource + self.bytesReadFromCache
47 bytesRead = property(getBytesRead)
48
50 total = self.bytesRead
51 if total == 0:
52 return 0.0
53 return float(self.bytesReadFromCache) / total
54 cacheReadRatio = property(getCacheReadRatio)
55
57 cs = self._stats
58 self._size = size
59 if cacheStatus == CACHE_HIT:
60 self._status = "cache-hit"
61 cs.cacheHitCount += 1
62 elif cacheStatus == TEMP_HIT:
63 cs.cacheHitCount += 1
64 cs.tempHitCount += 1
65 self._status = "temp-hit"
66 elif cacheStatus == CACHE_MISS:
67 cs.cacheMissCount += 1
68 if self._outdated:
69 self._status = "cache-outdate"
70 else:
71 self._status = "cache-miss"
72 cs._set("cache-hit-count", cs.cacheHitCount)
73 cs._set("temp-hit-count", cs.tempHitCount)
74 cs._set("cache-miss-count", cs.cacheMissCount)
75
77 self._outdated = True
78 cs = self._stats
79 cs.cacheOutdateCount += 1
80 cs._set("cache-outdate-count", cs.cacheOutdateCount)
81
82 - def onBytesRead(self, fromSource, fromCache, correction):
83 cs = self._stats
84 self.bytesReadFromSource += fromSource + correction
85 self.bytesReadFromCache += fromCache - correction
86 cs.bytesReadFromSource += fromSource + correction
87 cs.bytesReadFromCache += fromCache - correction
88
91
93 """
94 Provide the following log fields:
95 cache-status: value can be 'cache-miss', 'cache-outdate',
96 'cache-hit', or 'temp-hit'
97 cache-read: how many bytes where read from the cache for
98 this resource. the difference from resource-read
99 was read from the source file (network file system?)
100
101 The proportion read from cache and from source are adjusted
102 to take into account the file copy. It's done by remembering
103 how many bytes are copied at session level.
104 """
105 return {"cache-status": self._status,
106 "cache-read": self.bytesReadFromCache}
107
108
110
111 _updater = None
112 _callId = None
113
115
116 self._cacheUsage = 0
117 self._cacheUsageRatio = 0.0
118
119 self.cacheHitCount = 0
120 self.tempHitCount = 0
121 self.cacheMissCount = 0
122 self.cacheOutdateCount = 0
123 self.cleanupCount = 0
124
125 self.bytesReadFromSource = 0L
126 self.bytesReadFromCache = 0L
127
128 self.totalCopyCount = 0
129 self.currentCopyCount = 0
130 self.finishedCopyCount = 0
131 self.cancelledCopyCount = 0
132 self.bytesCopied = 0L
133 self._copyRatios = 0.0
134
136 self._updater = updater
137 if updater and (self._callId is None):
138 self._set("cache-usage-estimation", self._cacheUsage)
139 self._set("cache-usage-ratio-estimation", self._cacheUsageRatio)
140 self._set("cleanup-count", self.cleanupCount)
141 self._set("last-cleanup-time", time.time())
142 self._set("current-copy-count", self.currentCopyCount)
143 self._set("finished-copy-count", self.finishedCopyCount)
144 self._set("cancelled-copy-count", self.cancelledCopyCount)
145 self._set("mean-copy-ratio", self.meanCopyRatio)
146 self._set("mean-bytes-copied", self.meanBytesCopied)
147 self._update()
148
154
156 total = self.bytesReadFromSource + self.bytesReadFromCache
157 if total == 0:
158 return 0
159 return float(self.bytesReadFromCache) / total
160 cacheReadRatio = property(getCacheReadRatio)
161
163 if self.finishedCopyCount == 0:
164 return 0
165 return self.bytesCopied / self.finishedCopyCount
166 meanBytesCopied = property(getMeanBytesCopied)
167
169 if self.finishedCopyCount == 0:
170 return 0
171 return self._copyRatios / self.finishedCopyCount
172 meanCopyRatio = property(getMeanCopyRatio)
173
175 self._cacheUsage = usage
176 self._cacheUsageRatio = float(usage) / max
177 self._set("cache-usage-estimation", self._cacheUsage)
178 self._set("cache-usage-ratio-estimation", self._cacheUsageRatio)
179
181 self.cleanupCount += 1
182 self._set("cleanup-count", self.cleanupCount)
183 self._set("last-cleanup-time", time.time())
184
186 self.currentCopyCount += 1
187 self.totalCopyCount += 1
188 self._set("current-copy-count", self.currentCopyCount)
189
191 self.currentCopyCount -= 1
192 self.finishedCopyCount += 1
193 self.cancelledCopyCount += 1
194 self.bytesCopied += copied
195 self._copyRatios += float(copied) / size
196 self._set("current-copy-count", self.currentCopyCount)
197 self._set("finished-copy-count", self.finishedCopyCount)
198 self._set("cancelled-copy-count", self.cancelledCopyCount)
199 self._set("mean-copy-ratio", self.meanCopyRatio)
200 self._set("mean-bytes-copied", self.meanBytesCopied)
201
203 self.currentCopyCount -= 1
204 self.finishedCopyCount += 1
205 self.bytesCopied += size
206 self._copyRatios += 1.0
207 self._set("current-copy-count", self.currentCopyCount)
208 self._set("finished-copy-count", self.finishedCopyCount)
209 self._set("mean-copy-ratio", self.meanCopyRatio)
210 self._set("mean-bytes-copied", self.meanBytesCopied)
211
212 - def _set(self, key, value):
215
220
222 """
223 Statistic fields names:
224 CRR: Cache Read Ratio
225 CMC: Cache Miss Count
226 CHC: Cache Hit Count
227 THC: Temp Hit Count
228 COC: Cache Outdate Count
229 CCC: Cache Cleanup Count
230 CCU: Cache Current Usage
231 CUR: Cache Usage Ratio
232 PTC: coPy Total Count
233 PCC: coPy Current Count
234 PAC: coPy cAncellation Count
235 MCS: Mean Copy Size
236 MCR: Mean Copy Ratio
237 """
238 log.debug("stats-local-cache",
239 "CRR: %.4f; CMC: %d; CHC: %d; THC: %d; COC: %d; "
240 "CCC: %d; CCU: %d; CUR: %.5f; "
241 "PTC: %d; PCC: %d; PAC: %d; MCS: %d; MCR: %.4f",
242 self.cacheReadRatio, self.cacheMissCount,
243 self.cacheHitCount, self.tempHitCount,
244 self.cacheOutdateCount, self.cleanupCount,
245 self._cacheUsage, self._cacheUsageRatio,
246 self.totalCopyCount, self.currentCopyCount,
247 self.cancelledCopyCount, self.meanBytesCopied,
248 self.meanCopyRatio)
249