1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 """
19 bundles of files used to implement caching over the network
20 """
21
22 import StringIO
23 import errno
24 import os
25 import sys
26 import tempfile
27 import zipfile
28
29 from flumotion.common import errors, dag, python
30 from flumotion.common.python import makedirs
31
32 __all__ = ['Bundle', 'Bundler', 'Unbundler', 'BundlerBasket']
33 __version__ = "$Rev$"
34
35
38
39
41
42
43
44 try:
45 return os.rename(source, dest)
46 except WindowsError, e:
47 import winerror
48 if e.errno == winerror.ERROR_ALREADY_EXISTS:
49 os.unlink(source)
50
51
52 if sys.platform == 'win32':
53 rename = _win32Rename
54
55
57 """
58 I represent one file as managed by a bundler.
59 """
60
61 - def __init__(self, source, destination):
62 self.source = source
63 self.destination = destination
64 self._last_md5sum = None
65 self._last_timestamp = None
66 self.zipped = False
67
69 """
70 Calculate the md5sum of the given file.
71
72 @returns: the md5 sum a 32 character string of hex characters.
73 """
74 data = open(self.source, "r").read()
75 return python.md5(data).hexdigest()
76
78 """
79 @returns: the last modified timestamp for the file.
80 """
81 return os.path.getmtime(self.source)
82
84 """
85 Check if the file has changed since it was last checked.
86
87 @rtype: boolean
88 """
89
90
91
92
93 if not self.zipped:
94 return True
95
96 try:
97 timestamp = self.timestamp()
98 except OSError:
99 return True
100
101
102 if self._last_timestamp and timestamp <= self._last_timestamp:
103 return False
104 self._last_timestamp = timestamp
105
106
107 md5sum = self.md5sum()
108 if self._last_md5sum != md5sum:
109 self._last_md5sum = md5sum
110 return True
111
112 return False
113
114 - def pack(self, zip):
115 self._last_timestamp = self.timestamp()
116 self._last_md5sum = self.md5sum()
117 zip.write(self.source, self.destination)
118 self.zipped = True
119
120
122 """
123 I am a bundle of files, represented by a zip file and md5sum.
124 """
125
130
132 """
133 Set the bundle to the given data representation of the zip file.
134 """
135 self.zip = zip
136 self.md5sum = python.md5(self.zip).hexdigest()
137
139 """
140 Get the bundle's zip data.
141 """
142 return self.zip
143
144
146 """
147 I unbundle bundles by unpacking them in the given directory
148 under directories with the bundle's md5sum.
149 """
150
153
155 """
156 Return the full path where a bundle with the given name and md5sum
157 would be unbundled to.
158 """
159 return os.path.join(self._undir, name, md5sum)
160
166
168 """
169 Unbundle the given bundle.
170
171 @type bundle: L{flumotion.common.bundle.Bundle}
172
173 @rtype: string
174 @returns: the full path to the directory where it was unpacked
175 """
176 directory = self.unbundlePath(bundle)
177
178 filelike = StringIO.StringIO(bundle.getZip())
179 zipFile = zipfile.ZipFile(filelike, "r")
180 zipFile.testzip()
181
182 filepaths = zipFile.namelist()
183 for filepath in filepaths:
184 path = os.path.join(directory, filepath)
185 parent = os.path.split(path)[0]
186 try:
187 makedirs(parent)
188 except OSError, err:
189
190 if err.errno != errno.EEXIST or not os.path.isdir(parent):
191 raise
192 data = zipFile.read(filepath)
193
194
195 fd, tempname = tempfile.mkstemp(dir=parent)
196 handle = os.fdopen(fd, 'wb')
197 handle.write(data)
198 handle.close()
199 rename(tempname, path)
200 return directory
201
202
204 """
205 I bundle files into a bundle so they can be cached remotely easily.
206 """
207
209 """
210 Create a new bundle.
211 """
212 self._bundledFiles = {}
213 self.name = name
214 self._bundle = Bundle(name)
215
216 - def add(self, source, destination = None):
217 """
218 Add files to the bundle.
219
220 @param source: the path to the file to add to the bundle.
221 @param destination: a relative path to store this file in the bundle.
222 If unspecified, this will be stored in the top level.
223
224 @returns: the path the file got stored as
225 """
226 if destination == None:
227 destination = os.path.split(source)[1]
228 self._bundledFiles[source] = BundledFile(source, destination)
229 return destination
230
232 """
233 Bundle the files registered with the bundler.
234
235 @rtype: L{flumotion.common.bundle.Bundle}
236 """
237
238
239 if not self._bundle.getZip():
240 self._bundle.setZip(self._buildzip())
241 return self._bundle
242
243 update = False
244 for bundledFile in self._bundledFiles.values():
245 if bundledFile.hasChanged():
246 update = True
247 break
248
249 if update:
250 self._bundle.setZip(self._buildzip())
251
252 return self._bundle
253
254
255
256
258 filelike = StringIO.StringIO()
259 zipFile = zipfile.ZipFile(filelike, "w")
260 for bundledFile in self._bundledFiles.values():
261 bundledFile.pack(zipFile)
262 zipFile.close()
263 data = filelike.getvalue()
264 filelike.close()
265 return data
266
267
269 """
270 I manage bundlers that are registered through me.
271 """
272
274 """
275 Create a new bundler basket.
276 """
277 self._bundlers = {}
278
279 self._files = {}
280 self._imports = {}
281
282 self._graph = dag.DAG()
283
284 self._mtime = mtime
285
286
288 return self._mtime >= mtime
289
290 - def add(self, bundleName, source, destination=None):
291 """
292 Add files to the bundler basket for the given bundle.
293
294 @param bundleName: the name of the bundle this file is a part of
295 @param source: the path to the file to add to the bundle
296 @param destination: a relative path to store this file in the bundle.
297 If unspecified, this will be stored in the top level
298 """
299
300 if not bundleName in self._bundlers:
301 bundler = Bundler(bundleName)
302 self._bundlers[bundleName] = bundler
303 else:
304 bundler = self._bundlers[bundleName]
305
306
307 location = bundler.add(source, destination)
308 if location in self._files:
309 raise Exception("Cannot add %s to bundle %s, already in %s" % (
310 location, bundleName, self._files[location]))
311 self._files[location] = bundleName
312
313
314 package = None
315 if location.endswith('.py'):
316 package = location[:-3]
317 elif location.endswith('.pyc'):
318 package = location[:-4]
319
320 if package:
321 if package.endswith('__init__'):
322 package = os.path.split(package)[0]
323
324 package = ".".join(package.split('/'))
325 if package in self._imports:
326 raise Exception("Bundler %s already has import %s" % (
327 bundleName, package))
328 self._imports[package] = bundleName
329
330 - def depend(self, depender, *dependencies):
331 """
332 Make the given bundle depend on the other given bundles.
333
334 @type depender: string
335 @type dependencies: list of strings
336 """
337
338 if not self._graph.hasNode(depender):
339 self._graph.addNode(depender)
340 for dep in dependencies:
341 if not self._graph.hasNode(dep):
342 self._graph.addNode(dep)
343 self._graph.addEdge(depender, dep)
344
346 """
347 Return names of all the dependencies of this bundle, including this
348 bundle itself.
349 The dependencies are returned in a correct depending order.
350 """
351 if not bundlerName in self._bundlers:
352 raise errors.NoBundleError('Unknown bundle %s' % bundlerName)
353 elif not self._graph.hasNode(bundlerName):
354 return [bundlerName]
355 else:
356 return [bundlerName] + self._graph.getOffspring(bundlerName)
357
359 """
360 Return the bundle by name, or None if not found.
361 """
362 if bundlerName in self._bundlers:
363 return self._bundlers[bundlerName]
364 return None
365
367 """
368 Return the bundler name by import statement, or None if not found.
369 """
370 if importString in self._imports:
371 return self._imports[importString]
372 return None
373
375 """
376 Return the bundler name by filename, or None if not found.
377 """
378 if filename in self._files:
379 return self._files[filename]
380 return None
381
383 """
384 Get all bundler names.
385
386 @rtype: list of str
387 @returns: a list of all bundler names in this basket.
388 """
389 return self._bundlers.keys()
390
391
393 """
394 I am a bundler, with the extension that I can also bundle other
395 bundlers.
396
397 The effect is that when you call bundle() on a me, you get one
398 bundle with a union of all subbundlers' files, in addition to any
399 loose files that you added to me.
400 """
401
402 - def __init__(self, name='merged-bundle'):
405
407 """Add to me all of the files managed by another bundler.
408
409 @param bundler: The bundler whose files you want in this
410 bundler.
411 @type bundler: L{Bundler}
412 """
413 if bundler.name not in self._subbundlers:
414 self._subbundlers[bundler.name] = bundler
415 for bfile in bundler._files.values():
416 self.add(bfile.source, bfile.destination)
417
419 """
420 @returns: A list of all of the bundlers that have been added to
421 me.
422 """
423 return self._subbundlers.values()
424