1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 """objects and functions used in dealing with packages
19 """
20
21 import ihooks
22 import glob
23 import os
24 import sys
25
26 from twisted.python import rebuild, reflect
27
28 from flumotion.common import log, common
29 from flumotion.configure import configure
30
31 __version__ = "$Rev$"
32
33
35 """
36 I am overriding ihook's ModuleImporter's import_module() method to
37 accept (and ignore) the 'level' keyword argument that appeared in
38 the built-in __import__() function in python2.5.
39
40 While no built-in modules in python2.5 seem to use that keyword
41 argument, 'encodings' module in python2.6 does and so it breaks if
42 used together with ihooks.
43
44 I make no attempt to properly support the 'level' argument -
45 ihooks didn't make it into py3k, and the only use in python2.6
46 we've seen so far, in 'encodings', serves as a performance hint
47 and it seems that can be ignored with no difference in behaviour.
48 """
49
50 - def import_module(self, name, globals=None, locals=None, fromlist=None,
51 level=-1):
52
53 return ihooks.ModuleImporter.import_module(self, name, globals,
54 locals, fromlist)
55
56
58 """
59 I am an import Hooks object that makes sure that every package that gets
60 loaded has every necessary path in the module's __path__ list.
61
62 @type packager: L{Packager}
63 """
64 packager = None
65
87
88
90 """
91 I am an object through which package paths can be registered, to support
92 the partitioning of the module import namespace across bundles.
93 """
94
95 logCategory = 'packager'
96
98 self._paths = {}
99 self._packages = {}
100 self.install()
101
103 """
104 Install our custom importer that uses bundled packages.
105 """
106 self.debug('installing custom importer')
107 self._hooks = PackageHooks()
108 self._hooks.packager = self
109 if sys.version_info < (2, 6):
110 self._importer = ihooks.ModuleImporter()
111 else:
112 self.debug('python2.6 or later detected - using patched'
113 ' ModuleImporter')
114 self._importer = _PatchedModuleImporter()
115 self._importer.set_hooks(self._hooks)
116 self._importer.install()
117
119 """
120 Return all absolute paths to the top level of a tree from which
121 (part of) the given package name can be imported.
122 """
123 if packageName not in self._packages:
124 return None
125
126 return [self._paths[key] for key in self._packages[packageName]]
127
129 """
130 Register a given path as a path that can be imported from.
131 Used to support partition of bundled code or import code from various
132 uninstalled location.
133
134 sys.path will also be changed to include this, and remove references
135 to older packagePath's for the same bundle.
136
137 @param packagePath: path to add under which the module namespaces live,
138 (ending in an md5sum, for flumotion purposes)
139 @type packagePath: string
140 @param key a unique id for the package being registered
141 @type key: string
142 @param prefix: prefix of the packages to be considered
143 @type prefix: string
144 """
145
146 new = True
147 packagePath = os.path.abspath(packagePath)
148 if not os.path.exists(packagePath):
149 log.warning('bundle',
150 'registering a non-existing package path %s' % packagePath)
151
152 self.log('registering packagePath %s' % packagePath)
153
154
155 if key in self._paths:
156 oldPath = self._paths[key]
157 if packagePath == oldPath:
158 self.log('already registered %s for key %s' % (
159 packagePath, key))
160 return
161 new = False
162
163
164
165
166
167 if not os.path.isdir(packagePath):
168 log.warning('bundle', 'package path not a dir: %s',
169 packagePath)
170 packageNames = []
171 else:
172 packageNames = _findPackageCandidates(packagePath, prefix)
173
174 if not packageNames:
175 log.log('bundle',
176 'packagePath %s does not have candidates starting with %s' %
177 (packagePath, prefix))
178 return
179 packageNames.sort()
180
181 self.log('package candidates %r' % packageNames)
182
183 if not new:
184
185 log.log('bundle',
186 'replacing old path %s with new path %s for key %s' % (
187 oldPath, packagePath, key))
188
189 if oldPath in sys.path:
190 log.log('bundle',
191 'removing old packagePath %s from sys.path' % oldPath)
192 sys.path.remove(oldPath)
193
194
195 for keys in self._packages.values():
196 if key in keys:
197 keys.remove(key)
198
199 self._paths[key] = packagePath
200
201
202 if not packagePath in sys.path:
203 self.log('adding packagePath %s to sys.path' % packagePath)
204 sys.path.insert(0, packagePath)
205
206
207 for name in packageNames:
208 if name not in self._packages:
209 self._packages[name] = [key]
210 else:
211 self._packages[name].insert(0, key)
212
213 self.log('packagePath %s has packageNames %r' % (
214 packagePath, packageNames))
215
216
217 packageNames.reverse()
218
219 for packageName in packageNames:
220 if packageName not in sys.modules:
221 continue
222 self.log('fixing up %s ...' % packageName)
223
224
225 package = sys.modules.get(packageName)
226 for path in package.__path__:
227 if not new and path.startswith(oldPath):
228 self.log('%s.__path__ before remove %r' % (
229 packageName, package.__path__))
230 self.log('removing old %s from %s.__path__' % (
231 path, name))
232 package.__path__.remove(path)
233 self.log('%s.__path__ after remove %r' % (
234 packageName, package.__path__))
235
236
237
238
239
240 newPath = os.path.join(packagePath,
241 packageName.replace('.', os.sep))
242
243
244
245
246 if len(package.__path__) == 0:
247
248
249
250
251 self.debug('WARN: package %s does not have __path__ values' % (
252 packageName))
253 elif package.__path__[0] == newPath:
254 self.log('path %s already at start of %s.__path__' % (
255 newPath, packageName))
256 continue
257
258 if newPath in package.__path__:
259 package.__path__.remove(newPath)
260 self.log('moving %s to front of %s.__path__' % (
261 newPath, packageName))
262 else:
263 self.log('inserting new %s into %s.__path__' % (
264 newPath, packageName))
265 package.__path__.insert(0, newPath)
266
267
268
269
270
271
272
273
274 self.log('fixed up %s, __path__ %s ...' % (
275 packageName, package.__path__))
276
277
278
279 if not new:
280 self.log('finding end module candidates')
281 if not os.path.isdir(packagePath):
282 log.warning('bundle', 'package path not a dir: %s',
283 path)
284 moduleNames = []
285 else:
286 moduleNames = findEndModuleCandidates(packagePath, prefix)
287 self.log('end module candidates to rebuild: %r' % moduleNames)
288 for name in moduleNames:
289 if name in sys.modules:
290
291 self.log("rebuilding non-package module %s" % name)
292 try:
293 module = reflect.namedAny(name)
294 except AttributeError:
295 log.warning('bundle',
296 "could not reflect non-package module %s" % name)
297 continue
298
299 if hasattr(module, '__path__'):
300 self.log('rebuilding module %s with paths %r' % (name,
301 module.__path__))
302 rebuild.rebuild(module)
303
304
305
306 self.log('registered packagePath %s for key %s' % (packagePath, key))
307
309 """
310 Unregister all previously registered package paths, and uninstall
311 the custom importer.
312 """
313 for path in self._paths.values():
314 if path in sys.path:
315 self.log('removing packagePath %s from sys.path' % path)
316 sys.path.remove(path)
317 self._paths = {}
318 self._packages = {}
319 self.debug('uninstalling custom importer')
320 self._importer.uninstall()
321
322
324 """
325 I'm similar to os.listdir, but I work recursively and only return
326 directories containing python code.
327
328 @param path: the path
329 @type path: string
330 """
331 retval = []
332 try:
333 files = os.listdir(path)
334 except OSError:
335 pass
336 else:
337 for f in files:
338
339 p = os.path.join(path, f)
340 if os.path.isdir(p) and f != '.svn':
341 retval += _listDirRecursively(p)
342
343 if glob.glob(os.path.join(path, '*.py*')):
344 retval.append(path)
345
346 return retval
347
348
350 """
351 I'm similar to os.listdir, but I work recursively and only return
352 files representing python non-package modules.
353
354 @param path: the path
355 @type path: string
356
357 @rtype: list
358 @returns: list of files underneath the given path containing python code
359 """
360 retval = []
361
362
363 dirs = _listDirRecursively(path)
364
365 for directory in dirs:
366 pyfiles = glob.glob(os.path.join(directory, '*.py*'))
367 dontkeep = glob.glob(os.path.join(directory, '*__init__.py*'))
368 for f in dontkeep:
369 if f in pyfiles:
370 pyfiles.remove(f)
371
372 retval.extend(pyfiles)
373
374 return retval
375
376
378 """
379 I take a directory and return a list of candidate python packages
380 under that directory that start with the given prefix.
381 A package is a module containing modules; typically the directory
382 with the same name as the package contains __init__.py
383
384 @param path: the path
385 @type path: string
386 """
387
388
389 dirs = _listDirRecursively(os.path.join(path, prefix))
390
391
392 bundlePaths = [x[len(path) + 1:] for x in dirs]
393
394
395 bundlePaths = [path for path in bundlePaths if path.find('.svn') == -1]
396 bundlePaths = [path for path in bundlePaths if path.find('-') == -1]
397
398
399 bundlePackages = [".".join(x.split(os.path.sep)) for x in bundlePaths]
400
401
402
403 packages = {}
404 for name in bundlePackages:
405 packages[name] = 1
406 parts = name.split(".")
407 build = None
408 for p in parts:
409 if not build:
410 build = p
411 else:
412 build = build + "." + p
413 packages[build] = 1
414
415 bundlePackages = packages.keys()
416
417
418 bundlePackages.sort()
419
420 return bundlePackages
421
422
424 """
425 I take a directory and return a list of candidate python end modules
426 (i.e., non-package modules) for the given module prefix.
427
428 @param path: the path under which to search for end modules
429 @type path: string
430 @param prefix: module prefix to check candidates under
431 @type prefix: string
432 """
433 pathPrefix = "/".join(prefix.split("."))
434 files = _listPyFileRecursively(os.path.join(path, pathPrefix))
435
436
437 importPaths = [x[len(path) + 1:] for x in files]
438
439
440 importPaths = [path for path in importPaths if path.find('.svn') == -1]
441 importPaths = [path for path in importPaths if path.find('-') == -1]
442
443
444 endModules = [common.pathToModuleName(x) for x in importPaths]
445
446
447 endModules = [module for module in endModules
448 if module and module.startswith(prefix)]
449
450
451 endModules.sort()
452
453
454 res = {}
455 for b in endModules:
456 res[b] = 1
457
458 return res.keys()
459
460
461 __packager = None
462
463
475