Package flumotion :: Package common :: Module package
[hide private]

Source Code for Module flumotion.common.package

  1  # -*- Mode: Python; test-case-name: flumotion.test.test_common_package -*- 
  2  # vi:si:et:sw=4:sts=4:ts=4 
  3   
  4  # Flumotion - a streaming media server 
  5  # Copyright (C) 2004,2005,2006,2007,2008,2009 Fluendo, S.L. 
  6  # Copyright (C) 2010,2011 Flumotion Services, S.A. 
  7  # All rights reserved. 
  8  # 
  9  # This file may be distributed and/or modified under the terms of 
 10  # the GNU Lesser General Public License version 2.1 as published by 
 11  # the Free Software Foundation. 
 12  # This file is distributed without any warranty; without even the implied 
 13  # warranty of merchantability or fitness for a particular purpose. 
 14  # See "LICENSE.LGPL" in the source distribution for more information. 
 15  # 
 16  # Headers in this file shall remain intact. 
 17   
 18  """objects and functions used in dealing with packages 
 19  """ 
 20   
 21  import ihooks 
 22  import glob 
 23  import os 
 24  import sys 
 25   
 26  from twisted.python import rebuild, reflect 
 27   
 28  from flumotion.common import log, common 
 29  from flumotion.configure import configure 
 30   
 31  __version__ = "$Rev$" 
 32   
 33   
34 -class _PatchedModuleImporter(ihooks.ModuleImporter):
35 """ 36 I am overriding ihook's ModuleImporter's import_module() method to 37 accept (and ignore) the 'level' keyword argument that appeared in 38 the built-in __import__() function in python2.5. 39 40 While no built-in modules in python2.5 seem to use that keyword 41 argument, 'encodings' module in python2.6 does and so it breaks if 42 used together with ihooks. 43 44 I make no attempt to properly support the 'level' argument - 45 ihooks didn't make it into py3k, and the only use in python2.6 46 we've seen so far, in 'encodings', serves as a performance hint 47 and it seems that can be ignored with no difference in behaviour. 48 """ 49
50 - def import_module(self, name, globals=None, locals=None, fromlist=None, 51 level=-1):
52 # all we do is drop 'level' as ihooks don't support it, anyway 53 return ihooks.ModuleImporter.import_module(self, name, globals, 54 locals, fromlist)
55 56
57 -class PackageHooks(ihooks.Hooks):
58 """ 59 I am an import Hooks object that makes sure that every package that gets 60 loaded has every necessary path in the module's __path__ list. 61 62 @type packager: L{Packager} 63 """ 64 packager = None 65
66 - def load_package(self, name, filename, file=None):
67 # this is only ever called the first time a package is imported 68 log.log('packager', 'load_package %s' % name) 69 ret = ihooks.Hooks.load_package(self, name, filename, file) 70 71 m = sys.modules[name] 72 73 packagePaths = self.packager.getPathsForPackage(name) 74 if not packagePaths: 75 return ret 76 77 # get full paths to the package 78 paths = [os.path.join(path, name.replace('.', os.sep)) 79 for path in packagePaths] 80 for path in paths: 81 if not path in m.__path__: 82 log.log('packager', 'adding path %s for package %s' % ( 83 path, name)) 84 m.__path__.append(path) 85 86 return ret
87 88
89 -class Packager(log.Loggable):
90 """ 91 I am an object through which package paths can be registered, to support 92 the partitioning of the module import namespace across bundles. 93 """ 94 95 logCategory = 'packager' 96
97 - def __init__(self):
98 self._paths = {} # key -> package path registered with that key 99 self._packages = {} # package name -> keys for that package 100 self.install()
101
102 - def install(self):
103 """ 104 Install our custom importer that uses bundled packages. 105 """ 106 self.debug('installing custom importer') 107 self._hooks = PackageHooks() 108 self._hooks.packager = self 109 if sys.version_info < (2, 6): 110 self._importer = ihooks.ModuleImporter() 111 else: 112 self.debug('python2.6 or later detected - using patched' 113 ' ModuleImporter') 114 self._importer = _PatchedModuleImporter() 115 self._importer.set_hooks(self._hooks) 116 self._importer.install()
117
118 - def getPathsForPackage(self, packageName):
119 """ 120 Return all absolute paths to the top level of a tree from which 121 (part of) the given package name can be imported. 122 """ 123 if packageName not in self._packages: 124 return None 125 126 return [self._paths[key] for key in self._packages[packageName]]
127
128 - def registerPackagePath(self, packagePath, key, prefix=configure.PACKAGE):
129 """ 130 Register a given path as a path that can be imported from. 131 Used to support partition of bundled code or import code from various 132 uninstalled location. 133 134 sys.path will also be changed to include this, and remove references 135 to older packagePath's for the same bundle. 136 137 @param packagePath: path to add under which the module namespaces live, 138 (ending in an md5sum, for flumotion purposes) 139 @type packagePath: string 140 @param key a unique id for the package being registered 141 @type key: string 142 @param prefix: prefix of the packages to be considered 143 @type prefix: string 144 """ 145 146 new = True 147 packagePath = os.path.abspath(packagePath) 148 if not os.path.exists(packagePath): 149 log.warning('bundle', 150 'registering a non-existing package path %s' % packagePath) 151 152 self.log('registering packagePath %s' % packagePath) 153 154 # check if a packagePath for this bundle was already registered 155 if key in self._paths: 156 oldPath = self._paths[key] 157 if packagePath == oldPath: 158 self.log('already registered %s for key %s' % ( 159 packagePath, key)) 160 return 161 new = False 162 163 # Find the packages in the path and sort them, 164 # the following algorithm only works if they're sorted. 165 # By sorting the list we can ensure that a parent package 166 # is always processed before one of its children 167 if not os.path.isdir(packagePath): 168 log.warning('bundle', 'package path not a dir: %s', 169 packagePath) 170 packageNames = [] 171 else: 172 packageNames = _findPackageCandidates(packagePath, prefix) 173 174 if not packageNames: 175 log.log('bundle', 176 'packagePath %s does not have candidates starting with %s' % 177 (packagePath, prefix)) 178 return 179 packageNames.sort() 180 181 self.log('package candidates %r' % packageNames) 182 183 if not new: 184 # it already existed, and now it's a different path 185 log.log('bundle', 186 'replacing old path %s with new path %s for key %s' % ( 187 oldPath, packagePath, key)) 188 189 if oldPath in sys.path: 190 log.log('bundle', 191 'removing old packagePath %s from sys.path' % oldPath) 192 sys.path.remove(oldPath) 193 194 # clear this key from our name -> key cache 195 for keys in self._packages.values(): 196 if key in keys: 197 keys.remove(key) 198 199 self._paths[key] = packagePath 200 201 # put packagePath at the top of sys.path if not in there 202 if not packagePath in sys.path: 203 self.log('adding packagePath %s to sys.path' % packagePath) 204 sys.path.insert(0, packagePath) 205 206 # update our name->keys cache 207 for name in packageNames: 208 if name not in self._packages: 209 self._packages[name] = [key] 210 else: 211 self._packages[name].insert(0, key) 212 213 self.log('packagePath %s has packageNames %r' % ( 214 packagePath, packageNames)) 215 # since we want sub-modules to be fixed up before parent packages, 216 # we reverse the list 217 packageNames.reverse() 218 219 for packageName in packageNames: 220 if packageName not in sys.modules: 221 continue 222 self.log('fixing up %s ...' % packageName) 223 224 # the package is imported, so mess with __path__ and rebuild 225 package = sys.modules.get(packageName) 226 for path in package.__path__: 227 if not new and path.startswith(oldPath): 228 self.log('%s.__path__ before remove %r' % ( 229 packageName, package.__path__)) 230 self.log('removing old %s from %s.__path__' % ( 231 path, name)) 232 package.__path__.remove(path) 233 self.log('%s.__path__ after remove %r' % ( 234 packageName, package.__path__)) 235 236 # move the new path to the top 237 # insert at front because FLU_REGISTRY_PATH paths should override 238 # base components, and because subsequent reload() should prefer 239 # the latest registered path 240 newPath = os.path.join(packagePath, 241 packageName.replace('.', os.sep)) 242 243 # if path already at position 0, everything's fine 244 # if it's in there at another place, it needs to move to front 245 # if not in there, it needs to be put in front 246 if len(package.__path__) == 0: 247 # FIXME: this seems to happen to e.g. flumotion.component.base 248 # even when it was just rebuilt and had the __path__ set 249 # can be triggered by choosing a admin_gtk depending on 250 # the base admin_gtk where the base admin_gtk changes 251 self.debug('WARN: package %s does not have __path__ values' % ( 252 packageName)) 253 elif package.__path__[0] == newPath: 254 self.log('path %s already at start of %s.__path__' % ( 255 newPath, packageName)) 256 continue 257 258 if newPath in package.__path__: 259 package.__path__.remove(newPath) 260 self.log('moving %s to front of %s.__path__' % ( 261 newPath, packageName)) 262 else: 263 self.log('inserting new %s into %s.__path__' % ( 264 newPath, packageName)) 265 package.__path__.insert(0, newPath) 266 267 # Rebuilding these packages just to get __path__ fixed in 268 # seems not necessary - but re-enable it if it breaks 269 # self.log('rebuilding package %s from paths %r' % (packageName, 270 # package.__path__)) 271 # rebuild.rebuild(package) 272 # self.log('rebuilt package %s with paths %r' % (packageName, 273 # package.__path__)) 274 self.log('fixed up %s, __path__ %s ...' % ( 275 packageName, package.__path__)) 276 277 # now rebuild all non-package modules in this packagePath if this 278 # is not a new package 279 if not new: 280 self.log('finding end module candidates') 281 if not os.path.isdir(packagePath): 282 log.warning('bundle', 'package path not a dir: %s', 283 path) 284 moduleNames = [] 285 else: 286 moduleNames = findEndModuleCandidates(packagePath, prefix) 287 self.log('end module candidates to rebuild: %r' % moduleNames) 288 for name in moduleNames: 289 if name in sys.modules: 290 # fixme: isn't sys.modules[name] sufficient? 291 self.log("rebuilding non-package module %s" % name) 292 try: 293 module = reflect.namedAny(name) 294 except AttributeError: 295 log.warning('bundle', 296 "could not reflect non-package module %s" % name) 297 continue 298 299 if hasattr(module, '__path__'): 300 self.log('rebuilding module %s with paths %r' % (name, 301 module.__path__)) 302 rebuild.rebuild(module) 303 #if paths: 304 # module.__path__ = paths 305 306 self.log('registered packagePath %s for key %s' % (packagePath, key))
307
308 - def unregister(self):
309 """ 310 Unregister all previously registered package paths, and uninstall 311 the custom importer. 312 """ 313 for path in self._paths.values(): 314 if path in sys.path: 315 self.log('removing packagePath %s from sys.path' % path) 316 sys.path.remove(path) 317 self._paths = {} 318 self._packages = {} 319 self.debug('uninstalling custom importer') 320 self._importer.uninstall()
321 322
323 -def _listDirRecursively(path):
324 """ 325 I'm similar to os.listdir, but I work recursively and only return 326 directories containing python code. 327 328 @param path: the path 329 @type path: string 330 """ 331 retval = [] 332 try: 333 files = os.listdir(path) 334 except OSError: 335 pass 336 else: 337 for f in files: 338 # this only adds directories since files are not returned 339 p = os.path.join(path, f) 340 if os.path.isdir(p) and f != '.svn': 341 retval += _listDirRecursively(p) 342 343 if glob.glob(os.path.join(path, '*.py*')): 344 retval.append(path) 345 346 return retval
347 348
349 -def _listPyFileRecursively(path):
350 """ 351 I'm similar to os.listdir, but I work recursively and only return 352 files representing python non-package modules. 353 354 @param path: the path 355 @type path: string 356 357 @rtype: list 358 @returns: list of files underneath the given path containing python code 359 """ 360 retval = [] 361 362 # get all the dirs containing python code 363 dirs = _listDirRecursively(path) 364 365 for directory in dirs: 366 pyfiles = glob.glob(os.path.join(directory, '*.py*')) 367 dontkeep = glob.glob(os.path.join(directory, '*__init__.py*')) 368 for f in dontkeep: 369 if f in pyfiles: 370 pyfiles.remove(f) 371 372 retval.extend(pyfiles) 373 374 return retval
375 376
377 -def _findPackageCandidates(path, prefix=configure.PACKAGE):
378 """ 379 I take a directory and return a list of candidate python packages 380 under that directory that start with the given prefix. 381 A package is a module containing modules; typically the directory 382 with the same name as the package contains __init__.py 383 384 @param path: the path 385 @type path: string 386 """ 387 # this function also "guesses" candidate packages when __init__ is missing 388 # so a bundle with only a subpackage is also detected 389 dirs = _listDirRecursively(os.path.join(path, prefix)) 390 391 # chop off the base path to get a list of "relative" bundlespace paths 392 bundlePaths = [x[len(path) + 1:] for x in dirs] 393 394 # remove some common candidates, like .svn subdirs, or containing - 395 bundlePaths = [path for path in bundlePaths if path.find('.svn') == -1] 396 bundlePaths = [path for path in bundlePaths if path.find('-') == -1] 397 398 # convert paths to module namespace 399 bundlePackages = [".".join(x.split(os.path.sep)) for x in bundlePaths] 400 401 # now make sure that all parent packages for each package are listed 402 # as well 403 packages = {} 404 for name in bundlePackages: 405 packages[name] = 1 406 parts = name.split(".") 407 build = None 408 for p in parts: 409 if not build: 410 build = p 411 else: 412 build = build + "." + p 413 packages[build] = 1 414 415 bundlePackages = packages.keys() 416 417 # sort them so that depending packages are after higher-up packages 418 bundlePackages.sort() 419 420 return bundlePackages
421 422
423 -def findEndModuleCandidates(path, prefix=configure.PACKAGE):
424 """ 425 I take a directory and return a list of candidate python end modules 426 (i.e., non-package modules) for the given module prefix. 427 428 @param path: the path under which to search for end modules 429 @type path: string 430 @param prefix: module prefix to check candidates under 431 @type prefix: string 432 """ 433 pathPrefix = "/".join(prefix.split(".")) 434 files = _listPyFileRecursively(os.path.join(path, pathPrefix)) 435 436 # chop off the base path to get a list of "relative" import space paths 437 importPaths = [x[len(path) + 1:] for x in files] 438 439 # remove some common candidates, like .svn subdirs, or containing - 440 importPaths = [path for path in importPaths if path.find('.svn') == -1] 441 importPaths = [path for path in importPaths if path.find('-') == -1] 442 443 # convert paths to module namespace 444 endModules = [common.pathToModuleName(x) for x in importPaths] 445 446 # remove all not starting with prefix 447 endModules = [module for module in endModules 448 if module and module.startswith(prefix)] 449 450 # sort them so that depending packages are after higher-up packages 451 endModules.sort() 452 453 # make unique 454 res = {} 455 for b in endModules: 456 res[b] = 1 457 458 return res.keys()
459 460 # singleton factory function 461 __packager = None 462 463
464 -def getPackager():
465 """ 466 Return the (unique) packager. 467 468 @rtype: L{Packager} 469 """ 470 global __packager 471 if not __packager: 472 __packager = Packager() 473 474 return __packager
475