Package flumotion :: Package common :: Module bundle
[hide private]

Source Code for Module flumotion.common.bundle

  1  # -*- Mode: Python; test-case-name: flumotion.test.test_common_bundle -*- 
  2  # vi:si:et:sw=4:sts=4:ts=4 
  3   
  4  # Flumotion - a streaming media server 
  5  # Copyright (C) 2004,2005,2006,2007,2008,2009 Fluendo, S.L. 
  6  # Copyright (C) 2010,2011 Flumotion Services, S.A. 
  7  # All rights reserved. 
  8  # 
  9  # This file may be distributed and/or modified under the terms of 
 10  # the GNU Lesser General Public License version 2.1 as published by 
 11  # the Free Software Foundation. 
 12  # This file is distributed without any warranty; without even the implied 
 13  # warranty of merchantability or fitness for a particular purpose. 
 14  # See "LICENSE.LGPL" in the source distribution for more information. 
 15  # 
 16  # Headers in this file shall remain intact. 
 17   
 18  """ 
 19  bundles of files used to implement caching over the network 
 20  """ 
 21   
 22  import StringIO 
 23  import errno 
 24  import os 
 25  import sys 
 26  import tempfile 
 27  import zipfile 
 28   
 29  from flumotion.common import errors, dag, python 
 30  from flumotion.common.python import makedirs 
 31   
 32  __all__ = ['Bundle', 'Bundler', 'Unbundler', 'BundlerBasket'] 
 33  __version__ = "$Rev$" 
 34   
 35   
36 -def rename(source, dest):
37 return os.rename(source, dest)
38 39
40 -def _win32Rename(source, dest):
41 # rename a source to dest. 42 # ignores the destination if it already exists 43 # removes source if destination already exists 44 try: 45 return os.rename(source, dest) 46 except WindowsError, e: 47 import winerror 48 if e.errno == winerror.ERROR_ALREADY_EXISTS: 49 os.unlink(source)
50 51 52 if sys.platform == 'win32': 53 rename = _win32Rename 54 55
56 -class BundledFile:
57 """ 58 I represent one file as managed by a bundler. 59 """ 60
61 - def __init__(self, source, destination):
62 self.source = source 63 self.destination = destination 64 self._last_md5sum = None 65 self._last_timestamp = None 66 self.zipped = False
67
68 - def md5sum(self):
69 """ 70 Calculate the md5sum of the given file. 71 72 @returns: the md5 sum a 32 character string of hex characters. 73 """ 74 data = open(self.source, "r").read() 75 return python.md5(data).hexdigest()
76
77 - def timestamp(self):
78 """ 79 @returns: the last modified timestamp for the file. 80 """ 81 return os.path.getmtime(self.source)
82
83 - def hasChanged(self):
84 """ 85 Check if the file has changed since it was last checked. 86 87 @rtype: boolean 88 """ 89 90 # if it wasn't zipped yet, it needs zipping, so we pretend it 91 # was changed 92 # FIXME: move this out here 93 if not self.zipped: 94 return True 95 96 try: 97 timestamp = self.timestamp() 98 except OSError: 99 return True 100 # if file still has an old timestamp, it hasn't changed 101 # FIXME: looks bogus, shouldn't this check be != instead of <= ? 102 if self._last_timestamp and timestamp <= self._last_timestamp: 103 return False 104 self._last_timestamp = timestamp 105 106 # if the md5sum has changed, it has changed 107 md5sum = self.md5sum() 108 if self._last_md5sum != md5sum: 109 self._last_md5sum = md5sum 110 return True 111 112 return False
113
114 - def pack(self, zip):
115 self._last_timestamp = self.timestamp() 116 self._last_md5sum = self.md5sum() 117 zip.write(self.source, self.destination) 118 self.zipped = True
119 120
121 -class Bundle:
122 """ 123 I am a bundle of files, represented by a zip file and md5sum. 124 """ 125
126 - def __init__(self, name):
127 self.zip = None 128 self.md5sum = None 129 self.name = name
130
131 - def setZip(self, zip):
132 """ 133 Set the bundle to the given data representation of the zip file. 134 """ 135 self.zip = zip 136 self.md5sum = python.md5(self.zip).hexdigest()
137
138 - def getZip(self):
139 """ 140 Get the bundle's zip data. 141 """ 142 return self.zip
143 144
145 -class Unbundler:
146 """ 147 I unbundle bundles by unpacking them in the given directory 148 under directories with the bundle's md5sum. 149 """ 150
151 - def __init__(self, directory):
152 self._undir = directory
153
154 - def unbundlePathByInfo(self, name, md5sum):
155 """ 156 Return the full path where a bundle with the given name and md5sum 157 would be unbundled to. 158 """ 159 return os.path.join(self._undir, name, md5sum)
160
161 - def unbundlePath(self, bundle):
162 """ 163 Return the full path where this bundle will/would be unbundled to. 164 """ 165 return self.unbundlePathByInfo(bundle.name, bundle.md5sum)
166
167 - def unbundle(self, bundle):
168 """ 169 Unbundle the given bundle. 170 171 @type bundle: L{flumotion.common.bundle.Bundle} 172 173 @rtype: string 174 @returns: the full path to the directory where it was unpacked 175 """ 176 directory = self.unbundlePath(bundle) 177 178 filelike = StringIO.StringIO(bundle.getZip()) 179 zipFile = zipfile.ZipFile(filelike, "r") 180 zipFile.testzip() 181 182 filepaths = zipFile.namelist() 183 for filepath in filepaths: 184 path = os.path.join(directory, filepath) 185 parent = os.path.split(path)[0] 186 try: 187 makedirs(parent) 188 except OSError, err: 189 # Reraise error unless if it's an already existing 190 if err.errno != errno.EEXIST or not os.path.isdir(parent): 191 raise 192 data = zipFile.read(filepath) 193 194 # atomically write to path, see #373 195 fd, tempname = tempfile.mkstemp(dir=parent) 196 handle = os.fdopen(fd, 'wb') 197 handle.write(data) 198 handle.close() 199 rename(tempname, path) 200 return directory
201 202
203 -class Bundler:
204 """ 205 I bundle files into a bundle so they can be cached remotely easily. 206 """ 207
208 - def __init__(self, name):
209 """ 210 Create a new bundle. 211 """ 212 self._bundledFiles = {} # dictionary of BundledFile's indexed on path 213 self.name = name 214 self._bundle = Bundle(name)
215
216 - def add(self, source, destination = None):
217 """ 218 Add files to the bundle. 219 220 @param source: the path to the file to add to the bundle. 221 @param destination: a relative path to store this file in the bundle. 222 If unspecified, this will be stored in the top level. 223 224 @returns: the path the file got stored as 225 """ 226 if destination == None: 227 destination = os.path.split(source)[1] 228 self._bundledFiles[source] = BundledFile(source, destination) 229 return destination
230
231 - def bundle(self):
232 """ 233 Bundle the files registered with the bundler. 234 235 @rtype: L{flumotion.common.bundle.Bundle} 236 """ 237 # rescan files registered in the bundle, and check if we need to 238 # rebuild the internal zip 239 if not self._bundle.getZip(): 240 self._bundle.setZip(self._buildzip()) 241 return self._bundle 242 243 update = False 244 for bundledFile in self._bundledFiles.values(): 245 if bundledFile.hasChanged(): 246 update = True 247 break 248 249 if update: 250 self._bundle.setZip(self._buildzip()) 251 252 return self._bundle
253 254 # build the zip file containing the files registered in the bundle 255 # and return the zip file data 256
257 - def _buildzip(self):
258 filelike = StringIO.StringIO() 259 zipFile = zipfile.ZipFile(filelike, "w") 260 for bundledFile in self._bundledFiles.values(): 261 bundledFile.pack(zipFile) 262 zipFile.close() 263 data = filelike.getvalue() 264 filelike.close() 265 return data
266 267
268 -class BundlerBasket:
269 """ 270 I manage bundlers that are registered through me. 271 """ 272
273 - def __init__(self, mtime=None):
274 """ 275 Create a new bundler basket. 276 """ 277 self._bundlers = {} # bundler name -> bundle 278 279 self._files = {} # filename -> bundle name 280 self._imports = {} # import statements -> bundle name 281 282 self._graph = dag.DAG() 283 284 self._mtime = mtime # Registry modifcation time when the basket was
285 # created 286
287 - def isUptodate(self, mtime):
288 return self._mtime >= mtime
289
290 - def add(self, bundleName, source, destination=None):
291 """ 292 Add files to the bundler basket for the given bundle. 293 294 @param bundleName: the name of the bundle this file is a part of 295 @param source: the path to the file to add to the bundle 296 @param destination: a relative path to store this file in the bundle. 297 If unspecified, this will be stored in the top level 298 """ 299 # get the bundler and create it if need be 300 if not bundleName in self._bundlers: 301 bundler = Bundler(bundleName) 302 self._bundlers[bundleName] = bundler 303 else: 304 bundler = self._bundlers[bundleName] 305 306 # add the file to the bundle and register 307 location = bundler.add(source, destination) 308 if location in self._files: 309 raise Exception("Cannot add %s to bundle %s, already in %s" % ( 310 location, bundleName, self._files[location])) 311 self._files[location] = bundleName 312 313 # add possible imports from this file 314 package = None 315 if location.endswith('.py'): 316 package = location[:-3] 317 elif location.endswith('.pyc'): 318 package = location[:-4] 319 320 if package: 321 if package.endswith('__init__'): 322 package = os.path.split(package)[0] 323 324 package = ".".join(package.split('/')) # win32 fixme 325 if package in self._imports: 326 raise Exception("Bundler %s already has import %s" % ( 327 bundleName, package)) 328 self._imports[package] = bundleName
329
330 - def depend(self, depender, *dependencies):
331 """ 332 Make the given bundle depend on the other given bundles. 333 334 @type depender: string 335 @type dependencies: list of strings 336 """ 337 # note that a bundler doesn't necessarily need to be registered yet 338 if not self._graph.hasNode(depender): 339 self._graph.addNode(depender) 340 for dep in dependencies: 341 if not self._graph.hasNode(dep): 342 self._graph.addNode(dep) 343 self._graph.addEdge(depender, dep)
344
345 - def getDependencies(self, bundlerName):
346 """ 347 Return names of all the dependencies of this bundle, including this 348 bundle itself. 349 The dependencies are returned in a correct depending order. 350 """ 351 if not bundlerName in self._bundlers: 352 raise errors.NoBundleError('Unknown bundle %s' % bundlerName) 353 elif not self._graph.hasNode(bundlerName): 354 return [bundlerName] 355 else: 356 return [bundlerName] + self._graph.getOffspring(bundlerName)
357
358 - def getBundlerByName(self, bundlerName):
359 """ 360 Return the bundle by name, or None if not found. 361 """ 362 if bundlerName in self._bundlers: 363 return self._bundlers[bundlerName] 364 return None
365
366 - def getBundlerNameByImport(self, importString):
367 """ 368 Return the bundler name by import statement, or None if not found. 369 """ 370 if importString in self._imports: 371 return self._imports[importString] 372 return None
373
374 - def getBundlerNameByFile(self, filename):
375 """ 376 Return the bundler name by filename, or None if not found. 377 """ 378 if filename in self._files: 379 return self._files[filename] 380 return None
381
382 - def getBundlerNames(self):
383 """ 384 Get all bundler names. 385 386 @rtype: list of str 387 @returns: a list of all bundler names in this basket. 388 """ 389 return self._bundlers.keys()
390 391
392 -class MergedBundler(Bundler):
393 """ 394 I am a bundler, with the extension that I can also bundle other 395 bundlers. 396 397 The effect is that when you call bundle() on a me, you get one 398 bundle with a union of all subbundlers' files, in addition to any 399 loose files that you added to me. 400 """ 401
402 - def __init__(self, name='merged-bundle'):
403 Bundler.__init__(self, name) 404 self._subbundlers = {}
405
406 - def addBundler(self, bundler):
407 """Add to me all of the files managed by another bundler. 408 409 @param bundler: The bundler whose files you want in this 410 bundler. 411 @type bundler: L{Bundler} 412 """ 413 if bundler.name not in self._subbundlers: 414 self._subbundlers[bundler.name] = bundler 415 for bfile in bundler._files.values(): 416 self.add(bfile.source, bfile.destination)
417
418 - def getSubBundlers(self):
419 """ 420 @returns: A list of all of the bundlers that have been added to 421 me. 422 """ 423 return self._subbundlers.values()
424