Package flumotion :: Package common :: Module registry
[hide private]

Source Code for Module flumotion.common.registry

   1  # -*- Mode: Python; test-case-name: flumotion.test.test_registry -*- 
   2  # vi:si:et:sw=4:sts=4:ts=4 
   3   
   4  # Flumotion - a streaming media server 
   5  # Copyright (C) 2004,2005,2006,2007,2008,2009 Fluendo, S.L. 
   6  # Copyright (C) 2010,2011 Flumotion Services, S.A. 
   7  # All rights reserved. 
   8  # 
   9  # This file may be distributed and/or modified under the terms of 
  10  # the GNU Lesser General Public License version 2.1 as published by 
  11  # the Free Software Foundation. 
  12  # This file is distributed without any warranty; without even the implied 
  13  # warranty of merchantability or fitness for a particular purpose. 
  14  # See "LICENSE.LGPL" in the source distribution for more information. 
  15  # 
  16  # Headers in this file shall remain intact. 
  17   
  18  """parsing of registry, which holds component and bundle information 
  19  """ 
  20   
  21  import os 
  22  import stat 
  23  import errno 
  24  import sys 
  25  import tempfile 
  26  from StringIO import StringIO 
  27   
  28  from xml.sax import saxutils 
  29  from twisted.spread import pb 
  30  from twisted.python import runtime 
  31   
  32  from flumotion.common import common, log, errors, fxml, python 
  33  from flumotion.common.python import makedirs 
  34  from flumotion.common.bundle import BundlerBasket, MergedBundler 
  35  from flumotion.configure import configure 
  36   
  37  __all__ = ['ComponentRegistry', 'registry'] 
  38  __version__ = "$Rev$" 
  39   
  40  # Re-enable when reading the registry cache is lighter-weight, or we 
  41  # decide that it's a good idea, or something. See #799. 
  42  READ_CACHE = False 
  43  # Rank used when no rank is defined in the wizard entry 
  44  FLU_RANK_NONE = 0 
  45   
  46  _VALID_WIZARD_COMPONENT_TYPES = [ 
  47      'audio-producer', 
  48      'video-producer', 
  49      'muxer', 
  50      'audio-encoder', 
  51      'video-encoder', 
  52      'consumer', 
  53      ] 
  54   
  55  _VALID_WIZARD_PLUG_TYPES = [ 
  56      'http-consumer', 
  57      'httpserver-plug', 
  58      ] 
  59   
  60   
61 -def _getMTime(file):
62 return os.stat(file)[stat.ST_MTIME]
63 64
65 -class RegistryEntryScenario(pb.Copyable, pb.RemoteCopy):
66 """ 67 I represent a <scenario> entry in the registry 68 """ 69
70 - def __init__(self, type, description, base, entries):
71 """ 72 @param type: the type of this scenario 73 @type type: str 74 @param description: description of this scenario 75 @type description: str 76 @param base: base directory where this scenario is placed 77 @type base: str 78 @param entries: dict of entry point type -> entry 79 @type entries: dict of str -> L{RegistryEntryEntry} 80 """ 81 self.type = type 82 # we don't want to end up with the string "None" 83 self.description = description or "" 84 self.base = base 85 self.entries = entries
86
87 - def getEntries(self):
88 """ 89 Get the entries asociated with this scenario 90 91 @rtype: list of L{RegistryEntryEntry} 92 """ 93 return self.entries.values()
94
95 - def getEntryByType(self, type):
96 """ 97 Get the entry point for the given type of entry. 98 99 @param type: The type of the wanted entry. 100 @type type: string 101 102 @rtype: L{RegistryEntryEntry} 103 """ 104 return self.entries[type]
105
106 - def getType(self):
107 return self.type
108
109 - def getBase(self):
110 return self.base
111
112 - def getDescription(self):
113 return self.description
114 115 pb.setUnjellyableForClass(RegistryEntryScenario, RegistryEntryScenario) 116 117
118 -class RegistryEntryComponent(pb.Copyable, pb.RemoteCopy):
119 """ 120 I represent a <component> entry in the registry 121 """ 122 # RegistryEntryComponent has a constructor with a lot of arguments, 123 # but that's ok here. Allow it through pychecker. 124 __pychecker__ = 'maxargs=15' 125
126 - def __init__(self, filename, type, 127 source, description, base, properties, files, 128 entries, eaters, feeders, needs_sync, clock_priority, 129 sockets, wizards):
130 """ 131 @param filename: name of the XML file this component is parsed from 132 @type filename: str 133 @param properties: dict of name -> property 134 @type properties: dict of str -> L{RegistryEntryProperty} 135 @param files: list of files 136 @type files: list of L{RegistryEntryFile} 137 @param entries: dict of entry point type -> entry 138 @type entries: dict of str -> L{RegistryEntryEntry} 139 @param sockets: list of sockets supported by the component 140 @type sockets: list of str 141 @param wizards: list of wizard entries 142 @type wizards: list of L{RegistryEntryWizard} 143 """ 144 self.filename = filename 145 self.type = type 146 self.source = source 147 self.description = description 148 # we don't want to end up with the string "None" 149 if not self.description: 150 self.description = "" 151 self.base = base 152 self.properties = properties 153 self.files = files 154 self.entries = entries 155 self.eaters = eaters 156 self.feeders = feeders 157 self.needs_sync = needs_sync 158 self.clock_priority = clock_priority 159 self.sockets = sockets 160 self.wizards = wizards
161
162 - def getProperties(self):
163 """ 164 Get a list of all properties. 165 166 @rtype: list of L{RegistryEntryProperty} 167 """ 168 return self.properties.values()
169
170 - def hasProperty(self, name):
171 """ 172 Check if the component has a property with the given name. 173 """ 174 return name in self.properties.keys()
175
176 - def getFiles(self):
177 """ 178 @rtype: list of L{RegistryEntryFile} 179 """ 180 return self.files
181
182 - def getEntries(self):
183 return self.entries.values()
184
185 - def getEntryByType(self, type):
186 """ 187 Get the entry point for the given type of entry. 188 189 @type type: string 190 """ 191 return self.entries[type]
192
193 - def getGUIEntry(self):
194 if not self.files: 195 return 196 197 # FIXME: Handle multiple files 198 if len(self.files) > 1: 199 return 200 201 return self.files[0].getFilename()
202
203 - def getType(self):
204 return self.type
205
206 - def getBase(self):
207 return self.base
208
209 - def getDescription(self):
210 return self.description
211
212 - def getSource(self):
213 return self.source
214
215 - def getEaters(self):
216 return self.eaters
217
218 - def getFeeders(self):
219 return self.feeders
220
221 - def getNeedsSynchronization(self):
222 return self.needs_sync
223
224 - def getClockPriority(self):
225 return self.clock_priority
226
227 - def getSockets(self):
228 return self.sockets
229 pb.setUnjellyableForClass(RegistryEntryComponent, RegistryEntryComponent) 230 231
232 -class RegistryEntryPlug:
233 """ 234 I represent a <plug> entry in the registry 235 """ 236
237 - def __init__(self, filename, type, 238 description, socket, entries, properties, wizards):
239 """ 240 @param filename: name of the XML file this plug is parsed from 241 @type filename: str 242 @param type: the type of plug 243 @type type: str 244 @param description: the translatable description of the plug 245 @type description: str 246 @param socket: the fully qualified class name of the socket this 247 plug can be plugged in to 248 @type socket: str 249 @param entries: entry points for instantiating the plug 250 @type entries: list of L{RegistryEntryEntry} 251 @param properties: properties of the plug 252 @type properties: dict of str -> L{RegistryEntryProperty} 253 @param wizards: list of wizard entries 254 @type wizards: list of L{RegistryEntryWizard} 255 """ 256 self.filename = filename 257 self.type = type 258 self.description = description 259 self.socket = socket 260 self.entries = entries 261 self.properties = properties 262 self.wizards = wizards
263
264 - def getProperties(self):
265 """ 266 Get a list of all properties. 267 268 @rtype: list of L{RegistryEntryProperty} 269 """ 270 return self.properties.values()
271
272 - def hasProperty(self, name):
273 """ 274 Check if the component has a property with the given name. 275 """ 276 return name in self.properties.keys()
277
278 - def getEntryByType(self, type):
279 """ 280 Get the entry point for the given type of entry. 281 282 @type type: string 283 """ 284 return self.entries[type]
285
286 - def getEntry(self):
287 return self.entries['default']
288
289 - def getEntries(self):
290 return self.entries.values()
291
292 - def getType(self):
293 return self.type
294
295 - def getDescription(self):
296 return self.description
297
298 - def getSocket(self):
299 return self.socket
300 301
302 -class RegistryEntryBundle:
303 "This class represents a <bundle> entry in the registry" 304
305 - def __init__(self, name, project, under, dependencies, directories):
306 self.name = name 307 self.project = project 308 self.under = under 309 self.dependencies = dependencies 310 self.directories = directories
311
312 - def __repr__(self):
313 return '<Bundle name=%s>' % self.name
314
315 - def getName(self):
316 return self.name
317
318 - def getDependencies(self):
319 """ 320 @rtype: list of str 321 """ 322 return self.dependencies
323
324 - def getDirectories(self):
325 """ 326 @rtype: list of L{RegistryEntryBundleDirectory} 327 """ 328 return self.directories
329
330 - def getProject(self):
331 return self.project
332
333 - def getUnder(self):
334 return self.under
335
336 - def getBaseDir(self):
337 if self.project == configure.PACKAGE: 338 return getattr(configure, self.under) 339 340 from flumotion.project import project 341 return project.get(self.project, self.under)
342 343
344 -class RegistryEntryBundleDirectory:
345 "This class represents a <directory> entry in the registry" 346
347 - def __init__(self, name, files):
348 self.name = name 349 self.files = files
350
351 - def getName(self):
352 return self.name
353
354 - def getFiles(self):
355 return self.files
356 357
358 -class RegistryEntryBundleFilename:
359 "This class represents a <filename> entry in the registry" 360
361 - def __init__(self, location, relative):
362 self.location = location 363 self.relative = relative
364
365 - def getLocation(self):
366 return self.location
367
368 - def getRelative(self):
369 return self.relative
370 371
372 -class RegistryEntryProperty:
373 "This class represents a <property> entry in the registry" 374
375 - def __init__(self, name, type, description, 376 required=False, multiple=False):
377 self.name = name 378 self.type = type 379 self.description = description 380 # we don't want to end up with the string "None" 381 if not self.description: 382 self.description = "" 383 self.required = required 384 self.multiple = multiple
385
386 - def __repr__(self):
387 return '<Property name=%s>' % self.name
388
389 - def getName(self):
390 return self.name
391
392 - def getType(self):
393 return self.type
394
395 - def getDescription(self):
396 return self.description
397
398 - def isRequired(self):
399 return self.required
400
401 - def isMultiple(self):
402 return self.multiple
403 404
405 -class RegistryEntryCompoundProperty(RegistryEntryProperty):
406 "This class represents a <compound-property> entry in the registry" 407
408 - def __init__(self, name, description, properties, required=False, 409 multiple=False):
410 RegistryEntryProperty.__init__(self, name, 'compound', description, 411 required, multiple) 412 self.properties = properties
413
414 - def __repr__(self):
415 return '<Compound-property name=%s>' % self.name
416
417 - def getProperties(self):
418 """ 419 Get a list of all sub-properties. 420 421 @rtype: list of L{RegistryEntryProperty} 422 """ 423 return self.properties.values()
424
425 - def hasProperty(self, name):
426 """ 427 Check if the compound-property has a sub-property with the 428 given name. 429 """ 430 return name in self.properties
431 432
433 -class RegistryEntryFile:
434 "This class represents a <file> entry in the registry" 435
436 - def __init__(self, filename, type):
437 self.filename = filename 438 self.type = type
439
440 - def getName(self):
441 return os.path.basename(self.filename)
442
443 - def getType(self):
444 return self.type
445
446 - def getFilename(self):
447 return self.filename
448
449 - def isType(self, type):
450 return self.type == type
451 452
453 -class RegistryEntryEntry:
454 "This class represents a <entry> entry in the registry" 455
456 - def __init__(self, type, location, function):
457 self.type = type 458 self.location = location 459 self.function = function
460
461 - def getType(self):
462 return self.type
463
464 - def getLocation(self):
465 return self.location
466
467 - def getModuleName(self, base=None):
468 if base: 469 path = os.path.join(base, self.getLocation()) 470 else: 471 path = self.getLocation() 472 return common.pathToModuleName(path)
473
474 - def getFunction(self):
475 return self.function
476 477
478 -class RegistryEntryEater:
479 "This class represents a <eater> entry in the registry" 480
481 - def __init__(self, name, required=True, multiple=False):
482 self.name = name 483 self.required = required 484 self.multiple = multiple
485
486 - def getName(self):
487 return self.name
488
489 - def getRequired(self):
490 return self.required
491
492 - def getMultiple(self):
493 return self.multiple
494 495
496 -class RegistryEntryWizard(pb.Copyable):
497 "This class represents a <wizard> entry in the registry" 498
499 - def __init__(self, componentType, type, description, feeder, 500 eater, accepts, provides, rank=FLU_RANK_NONE):
501 self.componentType = componentType 502 self.type = type 503 self.description = description 504 self.feeder = feeder 505 self.eater = eater 506 self.accepts = accepts 507 self.provides = provides 508 self.rank = rank
509
510 - def __repr__(self):
511 return '<wizard %s type=%s, feeder=%s>' % (self.componentType, 512 self.type, self.feeder)
513 514
515 -class RegistryEntryWizardFormat(pb.Copyable):
516 """ 517 This class represents an <accept-format> or <provide-format> 518 entry in the registry 519 """ 520
521 - def __init__(self, media_type):
522 self.media_type = media_type
523 524
525 -class RegistryParser(fxml.Parser):
526 """ 527 Registry parser 528 529 I have two modes, one to parse registries and another one to parse 530 standalone component files. 531 532 For parsing registries use the parseRegistry function and for components 533 use parseRegistryFile. 534 535 I also have a list of all components and directories which the 536 registry uses (instead of saving its own copy) 537 """ 538
539 - def __init__(self):
540 self.clean()
541
542 - def clean(self):
543 self._components = {} 544 self._directories = {} # path -> RegistryDirectory 545 self._bundles = {} 546 self._plugs = {} 547 self._scenarios = {}
548
549 - def getComponents(self):
550 return self._components.values()
551
552 - def getComponent(self, name):
553 try: 554 return self._components[name] 555 except KeyError: 556 raise errors.UnknownComponentError("unknown component type:" 557 " %s" % (name, ))
558
559 - def getScenarios(self):
560 return self._scenarios.values()
561
562 - def getScenarioByType(self, type):
563 if type in self._scenarios: 564 return self._scenarios[type] 565 return None
566
567 - def getPlugs(self):
568 return self._plugs.values()
569
570 - def getPlug(self, name):
571 try: 572 return self._plugs[name] 573 except KeyError: 574 raise errors.UnknownPlugError("unknown plug type: %s" 575 % (name, ))
576
577 - def _parseComponents(self, node):
578 # <components> 579 # <component> 580 # </components> 581 582 components = {} 583 584 def addComponent(comp): 585 components[comp.getType()] = comp
586 587 parsers = {'component': (self._parseComponent, addComponent)} 588 self.parseFromTable(node, parsers) 589 590 return components
591
592 - def _parseComponent(self, node):
593 # <component type="..." base="..." _description="..."> 594 # <source> 595 # <eater> 596 # <feeder> 597 # <properties> 598 # <entries> 599 # <synchronization> 600 # <sockets> 601 # <wizard> 602 # </component> 603 604 # F0.10: remove description, require _description 605 componentType, baseDir, description, _description = \ 606 self.parseAttributes(node, 607 required=('type', 'base'), 608 optional=('description', '_description')) 609 610 # intltool-extract only translates attributes starting with _ 611 if description: 612 import warnings 613 warnings.warn( 614 "Please change '<component description=...'" 615 " to '<component _description=...' for %s" % componentType, 616 DeprecationWarning) 617 if _description: 618 description = _description 619 620 files = [] 621 source = fxml.Box(None) 622 entries = {} 623 eaters = [] 624 feeders = [] 625 synchronization = fxml.Box((False, 100)) 626 sockets = [] 627 properties = {} 628 wizards = [] 629 630 # Merge in options for inherit 631 #if node.hasAttribute('inherit'): 632 # base_type = str(node.getAttribute('inherit')) 633 # base = self.getComponent(base_type) 634 # for prop in base.getProperties(): 635 # properties[prop.getName()] = prop 636 637 parsers = { 638 'source': (self._parseSource, source.set), 639 'properties': (self._parseProperties, properties.update), 640 'files': (self._parseFiles, files.extend), 641 'entries': (self._parseEntries, entries.update), 642 'eater': (self._parseEater, eaters.append), 643 'feeder': (self._parseFeeder, feeders.append), 644 'synchronization': (self._parseSynchronization, 645 synchronization.set), 646 'sockets': (self._parseSockets, sockets.extend), 647 'wizard': (self._parseComponentWizard, wizards.append), 648 } 649 self.parseFromTable(node, parsers) 650 651 source = source.unbox() 652 needs_sync, clock_priority = synchronization.unbox() 653 654 return RegistryEntryComponent(self.filename, 655 componentType, source, description, 656 baseDir, properties, files, 657 entries, eaters, feeders, 658 needs_sync, clock_priority, 659 sockets, wizards)
660
661 - def _parseScenarios(self, node):
662 # <scenarios> 663 # <scenario> 664 # </scenarios> 665 666 scenarios = {} 667 668 def addScenario(scenario): 669 scenarios[scenario.getType()] = scenario
670 671 parsers = {'scenario': (self._parseScenario, addScenario)} 672 self.parseFromTable(node, parsers) 673 674 return scenarios 675
676 - def _parseScenario(self, node):
677 # <scenario type="..." base="..." _description="..."> 678 # <entries> 679 # </scenario> 680 681 scenarioType, baseDir, description = \ 682 self.parseAttributes(node, 683 required=('type', 'base'), 684 optional=('_description', )) 685 686 entries = {} 687 688 parsers = { 689 'entries': (self._parseEntries, entries.update), 690 } 691 692 self.parseFromTable(node, parsers) 693 694 return RegistryEntryScenario(scenarioType, description, 695 baseDir, entries)
696
697 - def _parseSource(self, node):
698 # <source location="..."/> 699 location, = self.parseAttributes(node, ('location', )) 700 return location
701
702 - def _parseProperty(self, node):
703 # <property name="..." type="" required="yes/no" multiple="yes/no"/> 704 # returns: RegistryEntryProperty 705 706 # F0.10: remove description, require _description 707 attrs = self.parseAttributes(node, required=('name', 'type'), 708 optional=('required', 'multiple', 'description', '_description')) 709 name, propertyType, required, multiple, description, _d = attrs 710 if description: 711 import warnings 712 warnings.warn("Please change '<property description=...'" 713 " to '<property _description=...' for %s" % name, 714 DeprecationWarning) 715 if _d: 716 description = _d 717 718 # see flumotion.common.config.parsePropertyValue 719 allowed = ('string', 'rawstring', 'int', 'long', 'bool', 720 'float', 'fraction') 721 if propertyType not in allowed: 722 raise fxml.ParserError( 723 "<property> %s's type is not one of %s" % ( 724 name, ", ".join(allowed))) 725 required = common.strToBool(required) 726 multiple = common.strToBool(multiple) 727 return RegistryEntryProperty(name, propertyType, description, 728 required=required, multiple=multiple)
729
730 - def _parseCompoundProperty(self, node):
731 # <compound-property name="..." required="yes/no" multiple="yes/no"> 732 # <property ... />* 733 # <compound-property ... >...</compound-property>* 734 # </compound-property> 735 # returns: RegistryEntryCompoundProperty 736 737 # F0.10: remove description, require _description 738 attrs = self.parseAttributes(node, required=('name', ), 739 optional=('required', 'multiple', 'description', '_description')) 740 name, required, multiple, description, _description = attrs 741 if description: 742 import warnings 743 warnings.warn("Please change '<compound-property description=...'" 744 " to '<compound-property _description=...' for %s" % name, 745 DeprecationWarning) 746 if _description: 747 description = _description 748 749 # see flumotion.common.config.parsePropertyValue 750 required = common.strToBool(required) 751 multiple = common.strToBool(multiple) 752 753 properties = {} 754 755 def addProperty(prop): 756 properties[prop.getName()] = prop
757 758 parsers = {'property': (self._parseProperty, addProperty), 759 'compound-property': (self._parseCompoundProperty, 760 addProperty)} 761 self.parseFromTable(node, parsers) 762 763 return RegistryEntryCompoundProperty(name, description, properties, 764 required=required, multiple=multiple) 765
766 - def _parseProperties(self, node):
767 # <properties> 768 # <property>* 769 # <compound-property>* 770 # </properties> 771 772 properties = {} 773 774 def addProperty(prop): 775 properties[prop.getName()] = prop
776 777 parsers = {'property': (self._parseProperty, addProperty), 778 'compound-property': (self._parseCompoundProperty, 779 addProperty)} 780 781 self.parseFromTable(node, parsers) 782 783 return properties 784
785 - def _parseFile(self, node):
786 # <file name="..." type=""/> 787 # returns: RegistryEntryFile 788 789 name, fileType = self.parseAttributes(node, ('name', 'type')) 790 directory = os.path.split(self.filename)[0] 791 filename = os.path.join(directory, name) 792 return RegistryEntryFile(filename, fileType)
793
794 - def _parseFiles(self, node):
795 # <files> 796 # <file> 797 # </files> 798 799 files = [] 800 parsers = {'file': (self._parseFile, files.append)} 801 802 self.parseFromTable(node, parsers) 803 804 return files
805
806 - def _parseSocket(self, node):
807 # <socket type=""/> 808 # returns: str of the type 809 810 socketType, = self.parseAttributes(node, ('type', )) 811 return socketType
812
813 - def _parseSockets(self, node):
814 # <sockets> 815 # <socket> 816 # </sockets> 817 818 sockets = [] 819 parsers = {'socket': (self._parseSocket, sockets.append)} 820 821 self.parseFromTable(node, parsers) 822 823 return sockets
824
825 - def _parseEntry(self, node):
826 attrs = self.parseAttributes(node, ('type', 'location', 'function')) 827 entryType, location, function = attrs 828 return RegistryEntryEntry(entryType, location, function)
829
830 - def _parseEntries(self, node):
831 # <entries> 832 # <entry> 833 # </entries> 834 # returns: dict of type -> entry 835 836 entries = {} 837 838 def addEntry(entry): 839 if entry.getType() in entries: 840 raise fxml.ParserError("entry %s already specified" 841 % entry.getType()) 842 entries[entry.getType()] = entry
843 844 parsers = {'entry': (self._parseEntry, addEntry)} 845 846 self.parseFromTable(node, parsers) 847 848 return entries 849
850 - def _parseEater(self, node):
851 # <eater name="..." [required="yes/no"] [multiple="yes/no"]/> 852 attrs = self.parseAttributes(node, ('name', ), 853 ('required', 'multiple')) 854 name, required, multiple = attrs 855 # only required defaults to True 856 required = common.strToBool(required or 'True') 857 multiple = common.strToBool(multiple) 858 859 return RegistryEntryEater(name, required, multiple)
860
861 - def _parseFeeder(self, node):
862 # <feeder name="..."/> 863 name, = self.parseAttributes(node, ('name', )) 864 return name
865
866 - def _parseSynchronization(self, node):
867 # <synchronization [required="yes/no"] [clock-priority="100"]/> 868 attrs = self.parseAttributes(node, (), ('required', 'clock-priority')) 869 required, clock_priority = attrs 870 required = common.strToBool(required) 871 clock_priority = int(clock_priority or '100') 872 return required, clock_priority
873
874 - def _parsePlugEntry(self, node):
875 attrs = self.parseAttributes(node, 876 ('location', 'function'), ('type', )) 877 location, function, entryType = attrs 878 if not entryType: 879 entryType = 'default' 880 return RegistryEntryEntry(entryType, location, function)
881
882 - def _parseDefaultPlugEntry(self, node):
883 return {'default': self._parsePlugEntry(node)}
884
885 - def _parsePlugEntries(self, node):
886 # <entries> 887 # <entry> 888 # </entries> 889 # returns: dict of type -> entry 890 891 entries = {} 892 893 def addEntry(entry): 894 if entry.getType() in entries: 895 raise fxml.ParserError("entry %s already specified" 896 % entry.getType()) 897 entries[entry.getType()] = entry
898 899 parsers = {'entry': (self._parsePlugEntry, addEntry)} 900 901 self.parseFromTable(node, parsers) 902 903 return entries 904
905 - def _parsePlug(self, node):
906 # <plug socket="..." type="..." _description="..."> 907 # <entries> 908 # <entry> 909 # <properties> 910 # <wizard> 911 # </plug> 912 913 # F0.10: make _description be required 914 plugType, socket, description = \ 915 self.parseAttributes(node, required=('type', 'socket'), 916 optional=('_description', )) 917 918 if not description: 919 import warnings 920 warnings.warn( 921 "Please add '_description=...' attribute to plug '%s'" % 922 plugType, 923 DeprecationWarning) 924 description = 'TODO' 925 926 entries = {} 927 properties = {} 928 wizards = [] 929 930 parsers = { 931 'entries': (self._parsePlugEntries, entries.update), 932 # backwards compatibility 933 'entry': (self._parseDefaultPlugEntry, entries.update), 934 'properties': (self._parseProperties, properties.update), 935 'wizard': (self._parsePlugWizard, wizards.append), 936 } 937 938 self.parseFromTable(node, parsers) 939 940 if not 'default' in entries: 941 raise fxml.ParserError( 942 "<plug> %s needs a default <entry>" % plugType) 943 944 return RegistryEntryPlug(self.filename, plugType, description, 945 socket, entries, properties, 946 wizards)
947
948 - def _parsePlugs(self, node):
949 # <plugs> 950 # <plug> 951 # </plugs> 952 953 self.checkAttributes(node) 954 955 plugs = {} 956 957 def addPlug(plug): 958 plugs[plug.getType()] = plug
959 960 parsers = {'plug': (self._parsePlug, addPlug)} 961 self.parseFromTable(node, parsers) 962 963 return plugs 964 965 ## Component registry specific functions 966
967 - def parseRegistryFile(self, file):
968 """ 969 @param file: The file to parse, either as an open file object, 970 or as the name of a file to open. 971 @type file: str or file. 972 """ 973 if isinstance(file, basestring): 974 self.filename = file 975 else: 976 self.filename = getattr(file, 'name', '<string>') 977 root = self.getRoot(file) 978 node = root.documentElement 979 980 if node.nodeName != 'registry': 981 # ignore silently, since this function is used to parse all 982 # .xml files encountered 983 self.debug('%s does not have registry as root tag', self.filename) 984 return 985 986 # shouldn't have <directories> elements in registry fragments 987 self._parseRoot(node, disallowed=['directories']) 988 root.unlink()
989
990 - def _parseBundles(self, node):
991 # <bundles> 992 # <bundle> 993 # </bundles> 994 995 bundles = {} 996 997 def addBundle(bundle): 998 bundles[bundle.getName()] = bundle
999 1000 parsers = {'bundle': (self._parseBundle, addBundle)} 1001 self.parseFromTable(node, parsers) 1002 1003 return bundles 1004
1005 - def _parseBundle(self, node):
1006 # <bundle name="..."> 1007 # <dependencies> 1008 # <directories> 1009 # </bundle> 1010 1011 attrs = self.parseAttributes(node, ('name', ), ('project', 'under')) 1012 name, project, under = attrs 1013 project = project or configure.PACKAGE 1014 under = under or 'pythondir' 1015 1016 dependencies = [] 1017 directories = [] 1018 1019 parsers = {'dependencies': (self._parseBundleDependencies, 1020 dependencies.extend), 1021 'directories': (self._parseBundleDirectories, 1022 directories.extend)} 1023 self.parseFromTable(node, parsers) 1024 1025 return RegistryEntryBundle(name, project, under, 1026 dependencies, directories)
1027
1028 - def _parseBundleDependency(self, node):
1029 name, = self.parseAttributes(node, ('name', )) 1030 return name
1031
1032 - def _parseBundleDependencies(self, node):
1033 # <dependencies> 1034 # <dependency name=""> 1035 # </dependencies> 1036 dependencies = [] 1037 1038 parsers = {'dependency': (self._parseBundleDependency, 1039 dependencies.append)} 1040 self.parseFromTable(node, parsers) 1041 1042 return dependencies
1043
1044 - def _parseBundleDirectories(self, node):
1045 # <directories> 1046 # <directory> 1047 # </directories> 1048 directories = [] 1049 1050 parsers = {'directory': (self._parseBundleDirectory, 1051 directories.append)} 1052 self.parseFromTable(node, parsers) 1053 1054 return directories
1055
1056 - def _parseBundleDirectoryFilename(self, node, name):
1057 attrs = self.parseAttributes(node, ('location', ), ('relative', )) 1058 location, relative = attrs 1059 1060 if not relative: 1061 relative = os.path.join(name, location) 1062 1063 return RegistryEntryBundleFilename(location, relative)
1064
1065 - def _parseBundleDirectory(self, node):
1066 # <directory name=""> 1067 # <filename location="" [ relative="" ] > 1068 # </directory> 1069 name, = self.parseAttributes(node, ('name', )) 1070 1071 filenames = [] 1072 1073 def parseFilename(node): 1074 return self._parseBundleDirectoryFilename(node, name)
1075 1076 parsers = {'filename': (parseFilename, filenames.append)} 1077 self.parseFromTable(node, parsers) 1078 1079 return RegistryEntryBundleDirectory(name, filenames) 1080 1081 ## Base registry specific functions 1082
1083 - def parseRegistry(self, file):
1084 """ 1085 @param file: The file to parse, either as an open file object, 1086 or as the name of a file to open. 1087 @type file: str or file. 1088 """ 1089 if isinstance(file, basestring): 1090 self.filename = file 1091 else: 1092 self.filename = getattr(file, 'name', '<string>') 1093 root = self.getRoot(file) 1094 self._parseRoot(root.documentElement) 1095 root.unlink()
1096
1097 - def getDirectories(self):
1098 return self._directories.values()
1099
1100 - def getDirectory(self, name):
1101 return self._directories[name]
1102
1103 - def addDirectory(self, directory):
1104 """ 1105 Add a registry path object to the parser. 1106 1107 @type directory: {RegistryDirectory} 1108 """ 1109 self._directories[directory.getPath()] = directory
1110
1111 - def removeDirectoryByPath(self, path):
1112 """ 1113 Remove a directory from the parser given the path. 1114 Used when the path does not actually contain any registry information. 1115 """ 1116 if path in self._directories.keys(): 1117 del self._directories[path]
1118
1119 - def _parseRoot(self, node, disallowed=None):
1120 # <components>...</components>* 1121 # <plugs>...</plugs>* 1122 # <directories>...</directories>* 1123 # <bundles>...</bundles>* 1124 # <scenarios>...</scenarios>* 1125 parsers = {'components': (self._parseComponents, 1126 self._components.update), 1127 'directories': (self._parseDirectories, 1128 self._directories.update), 1129 'bundles': (self._parseBundles, self._bundles.update), 1130 'plugs': (self._parsePlugs, self._plugs.update), 1131 'scenarios': (self._parseScenarios, self._scenarios.update)} 1132 1133 if disallowed: 1134 for k in disallowed: 1135 del parsers[k] 1136 1137 self.parseFromTable(node, parsers)
1138
1139 - def _parseDirectories(self, node):
1140 # <directories> 1141 # <directory> 1142 # </directories> 1143 1144 directories = {} 1145 1146 def addDirectory(d): 1147 directories[d.getPath()] = d
1148 1149 parsers = {'directory': (self._parseDirectory, addDirectory)} 1150 self.parseFromTable(node, parsers) 1151 1152 return directories 1153
1154 - def _parseDirectory(self, node):
1155 # <directory filename="..."/> 1156 filename, = self.parseAttributes(node, ('filename', )) 1157 return RegistryDirectory(filename)
1158
1159 - def _parseComponentWizard(self, node):
1160 return self._parseWizard(node, _VALID_WIZARD_COMPONENT_TYPES)
1161
1162 - def _parsePlugWizard(self, node):
1163 return self._parseWizard(node, _VALID_WIZARD_PLUG_TYPES)
1164
1165 - def _parseWizard(self, node, validTypes):
1166 # <wizard type="..." _description=" " feeder="..." eater="..."]/> 1167 # 1168 # NOTE: We are using _description with the leading underscore for 1169 # the case of intltool, it is not possible for it to pickup 1170 # translated attributes otherwise. Ideally we would use another 1171 # tool so we can avoid underscores in our xml schema. 1172 attrs = self.parseAttributes(node, 1173 ('type', '_description'), 1174 ('feeder', 'eater', 'rank')) 1175 wizardType, description, feeder, eater, rank = attrs 1176 1177 accepts = [] 1178 provides = [] 1179 parsers = { 1180 'accept-format': (self._parseAcceptFormat, 1181 lambda n: accepts.append(n)), 1182 'provide-format': (self._parseProvideFormat, 1183 lambda n: provides.append(n)), 1184 } 1185 self.parseFromTable(node, parsers) 1186 1187 parent_type = node.parentNode.getAttribute('type') 1188 1189 if not wizardType in validTypes: 1190 raise fxml.ParserError( 1191 "<wizard>'s type attribute is %s must be one of %s" % ( 1192 parent_type, 1193 ', '.join(validTypes))) 1194 rank = int(rank or FLU_RANK_NONE) 1195 isProducer = wizardType.endswith('-producer') 1196 isEncoder = wizardType.endswith('-encoder') 1197 isMuxer = (wizardType == 'muxer') 1198 isConsumer = wizardType.endswith('-consumer') 1199 1200 err = None 1201 # Producers and Encoders cannot have provided 1202 if accepts and (isProducer or isEncoder): 1203 err = ('<wizard type="%s"> does not allow an accepted ' 1204 'media-type.') % (parent_type, ) 1205 # Encoders, Muxers and Consumers must have an accepted 1206 elif not accepts and (isMuxer or isConsumer): 1207 err = ('<wizard type="%s"> requires at least one accepted ' 1208 'media-type.') % (parent_type, ) 1209 # Producers and Consumers cannot have provided 1210 elif provides and (isProducer or isConsumer): 1211 err = ('<wizard type="%s"> does not allow a provided ' 1212 'media-type.') % (parent_type, ) 1213 # Producers, Encoders and Muxers must have exactly one provided 1214 if len(provides) != 1 and (isEncoder or isMuxer): 1215 err = ('<wizard type="%s"> requires exactly one provided ' 1216 'media-type.') % (parent_type, ) 1217 1218 if err: 1219 raise fxml.ParserError(err) 1220 1221 return RegistryEntryWizard(parent_type, wizardType, description, 1222 feeder, eater, accepts, provides, rank)
1223
1224 - def _parseAcceptFormat(self, node):
1225 # <accept-format media-type="..."/> 1226 media_type, = self.parseAttributes(node, ('media-type', )) 1227 return RegistryEntryWizardFormat(media_type)
1228
1229 - def _parseProvideFormat(self, node):
1230 # <provide-format media-type="..."/> 1231 media_type, = self.parseAttributes(node, ('media-type', )) 1232 return RegistryEntryWizardFormat(media_type)
1233 1234 1235 # FIXME: filename -> path 1236 1237
1238 -class RegistryDirectory(log.Loggable):
1239 """ 1240 I represent a directory under a path managed by the registry. 1241 I can be queried for a list of partial registry .xml files underneath 1242 the given path, under the given prefix. 1243 """ 1244
1245 - def __init__(self, path, prefix=configure.PACKAGE):
1246 self._path = path 1247 self._prefix = prefix 1248 scanPath = os.path.join(path, prefix) 1249 self._files, self._dirs = self._getFileLists(scanPath)
1250
1251 - def __repr__(self):
1252 return "<RegistryDirectory %s>" % self._path
1253
1254 - def _getFileLists(self, root):
1255 """ 1256 Get all files ending in .xml from all directories under the given root. 1257 1258 @type root: string 1259 @param root: the root directory under which to search 1260 1261 @returns: a list of .xml files, relative to the given root directory 1262 """ 1263 files = [] 1264 dirs = [] 1265 1266 if os.path.exists(root): 1267 try: 1268 directory_files = os.listdir(root) 1269 except OSError, e: 1270 if e.errno == errno.EACCES: 1271 return files, dirs 1272 else: 1273 raise 1274 1275 dirs.append(root) 1276 1277 for entry in directory_files: 1278 path = os.path.join(root, entry) 1279 # if it's a .xml file, then add it to the list 1280 if not os.path.isdir(path): 1281 if path.endswith('.xml'): 1282 files.append(path) 1283 # if it's a directory and not an svn directory, then get 1284 # its files and add them 1285 elif entry != '.svn': 1286 newFiles, newDirs = self._getFileLists(path) 1287 files.extend(newFiles) 1288 dirs.extend(newDirs) 1289 1290 return files, dirs
1291
1292 - def rebuildNeeded(self, mtime):
1293 1294 def _rebuildNeeded(f): 1295 try: 1296 if _getMTime(f) > mtime: 1297 self.debug("Path %s changed since registry last " 1298 "scanned", f) 1299 return True 1300 return False 1301 except OSError: 1302 self.debug("Failed to stat file %s, need to rescan", f) 1303 return True
1304 1305 for f in self._files: 1306 if _rebuildNeeded(f): 1307 return True 1308 for f in self._dirs: 1309 if _rebuildNeeded(f): 1310 return True 1311 return False
1312
1313 - def getFiles(self):
1314 """ 1315 Return a list of all .xml registry files underneath this registry 1316 path. 1317 """ 1318 return self._files
1319
1320 - def getPath(self):
1321 return self._path
1322 1323
1324 -class RegistryWriter(log.Loggable):
1325
1326 - def __init__(self, components, plugs, bundles, directories):
1327 """ 1328 @param components: components to write 1329 @type components: list of L{RegistryEntryComponent} 1330 @param plugs: plugs to write 1331 @type plugs: list of L{RegistryEntryPlug} 1332 @param bundles: bundles to write 1333 @type bundles: list of L{RegistryEntryBundle} 1334 @param directories: directories to write 1335 @type directories: list of L{RegistryEntryBundleDirectory} 1336 """ 1337 self.components = components 1338 self.plugs = plugs 1339 self.bundles = bundles 1340 self.directories = directories
1341
1342 - def dump(self, fd):
1343 """ 1344 Dump the cache of components to the given opened file descriptor. 1345 1346 @type fd: integer 1347 @param fd: open file descriptor to write to 1348 """ 1349 1350 def w(i, msg): 1351 print >> fd, ' '*i + msg
1352 1353 def e(attr): 1354 return saxutils.quoteattr(attr)
1355 1356 def _dump_proplist(i, proplist, ioff=2): 1357 for prop in proplist: 1358 if isinstance(prop, RegistryEntryCompoundProperty): 1359 _dump_compound(i, prop) 1360 else: 1361 w(i, ('<property name="%s" type="%s"' 1362 % (prop.getName(), prop.getType()))) 1363 w(i, (' _description=%s' 1364 % (e(prop.getDescription()), ))) 1365 w(i, (' required="%s" multiple="%s"/>' 1366 % (prop.isRequired(), prop.isMultiple()))) 1367 1368 def _dump_compound(i, cprop, ioff=2): 1369 w(i, ('<compound-property name="%s"' % (cprop.getName(), ))) 1370 w(i, (' _description=%s' 1371 % (e(cprop.getDescription()), ))) 1372 w(i, (' required="%s" multiple="%s">' 1373 % (cprop.isRequired(), cprop.isMultiple()))) 1374 _dump_proplist(i + ioff, cprop.getProperties()) 1375 w(i, ('</compound-property>')) 1376 1377 def _dump_entries(i, entries): 1378 if not entries: 1379 return 1380 1381 w(i, '<entries>') 1382 for entry in entries: 1383 w(i+2, '<entry type="%s" location="%s" function="%s"/>' % ( 1384 entry.getType(), 1385 entry.getLocation(), 1386 entry.getFunction())) 1387 w(i, '</entries>') 1388 1389 w(0, '<registry>') 1390 w(0, '') 1391 1392 # Write components 1393 w(2, '<components>') 1394 w(0, '') 1395 for component in self.components: 1396 w(4, '<component type="%s" base="%s"' % ( 1397 component.getType(), component.getBase())) 1398 w(4, ' _description=%s>' 1399 % (e(component.getDescription()), )) 1400 1401 w(6, '<source location="%s"/>' % component.getSource()) 1402 for x in component.getEaters(): 1403 w(6, '<eater name="%s" required="%s" multiple="%s"/>' 1404 % (x.getName(), x.getRequired() and "yes" or "no", 1405 x.getMultiple() and "yes" or "no")) 1406 for x in component.getFeeders(): 1407 w(6, '<feeder name="%s"/>' % x) 1408 w(6, '<synchronization required="%s" clock-priority="%d"/>' 1409 % (component.getNeedsSynchronization() and "yes" or "no", 1410 component.getClockPriority())) 1411 1412 sockets = component.getSockets() 1413 if sockets: 1414 w(6, '<sockets>') 1415 for socket in sockets: 1416 w(8, '<socket type="%s"/>' % socket) 1417 w(6, '</sockets>') 1418 1419 w(6, '<properties>') 1420 _dump_proplist(8, component.getProperties()) 1421 w(6, '</properties>') 1422 1423 for wizard in component.wizards: 1424 rank = '' 1425 if wizard.rank: 1426 rank = ' rank="%d"' % wizard.rank 1427 w(6, '<wizard type="%s" _description="%s" feeder="%s"%s>' % ( 1428 wizard.type, 1429 e(wizard.description), 1430 wizard.feeder, 1431 rank)) 1432 for accept in wizard.accepts: 1433 w(8, '<accept-format media-type="%s"/>' % ( 1434 accept.media_type)) 1435 for provide in wizard.provides: 1436 w(8, '<provide-format media-type="%s"/>' % ( 1437 provide.media_type)) 1438 w(6, '</wizard>') 1439 1440 registryEntryFiles = component.getFiles() 1441 if registryEntryFiles: 1442 w(6, '<files>') 1443 for entryFile in registryEntryFiles: 1444 w(8, '<file name="%s" type="%s"/>' % ( 1445 entryFile.getName(), 1446 entryFile.getType())) 1447 w(6, '</files>') 1448 1449 _dump_entries(6, component.getEntries()) 1450 1451 w(4, '</component>') 1452 w(0, '') 1453 1454 w(2, '</components>') 1455 w(0, '') 1456 1457 # Write plugs 1458 w(2, '<plugs>') 1459 w(0, '') 1460 for plug in self.plugs: 1461 w(4, '<plug type="%s" socket="%s" _description="%s">' 1462 % (plug.getType(), plug.getSocket(), plug.getDescription())) 1463 1464 _dump_entries(6, plug.getEntries()) 1465 1466 w(6, '<properties>') 1467 _dump_proplist(8, plug.getProperties()) 1468 w(6, '</properties>') 1469 1470 w(4, '</plug>') 1471 w(0, '') 1472 1473 w(2, '</plugs>') 1474 w(0, '') 1475 1476 # bundles 1477 w(2, '<bundles>') 1478 for bundle in self.bundles: 1479 w(4, '<bundle name="%s" under="%s" project="%s">' % ( 1480 bundle.getName(), bundle.getUnder(), bundle.getProject())) 1481 1482 dependencies = bundle.getDependencies() 1483 if dependencies: 1484 w(6, '<dependencies>') 1485 for dependency in dependencies: 1486 w(8, '<dependency name="%s"/>' % dependency) 1487 w(6, '</dependencies>') 1488 1489 bundleDirectories = bundle.getDirectories() 1490 if bundleDirectories: 1491 w(6, '<directories>') 1492 for directory in bundleDirectories: 1493 w(8, '<directory name="%s">' % directory.getName()) 1494 for filename in directory.getFiles(): 1495 w(10, '<filename location="%s" relative="%s"/>' % ( 1496 filename.getLocation(), filename.getRelative())) 1497 w(8, '</directory>') 1498 w(6, '</directories>') 1499 1500 w(4, '</bundle>') 1501 w(0, '') 1502 w(2, '</bundles>') 1503 1504 1505 # Directories 1506 directories = self.directories 1507 if directories: 1508 w(2, '<directories>') 1509 w(0, '') 1510 for d in directories: 1511 w(4, '<directory filename="%s"/>' % d.getPath()) 1512 w(2, '</directories>') 1513 w(0, '') 1514 1515 w(0, '</registry>') 1516 1517
1518 -class ComponentRegistry(log.Loggable):
1519 """Registry, this is normally not instantiated.""" 1520 1521 logCategory = 'registry' 1522 defaultCachePath = os.path.join(configure.registrydir, 'registry.xml') 1523
1524 - def __init__(self, paths=None, prefix=configure.PACKAGE, 1525 cachePath=defaultCachePath, seconds=runtime.seconds):
1526 if paths is not None: 1527 self._paths = paths 1528 else: 1529 self._paths = self._getRegistryPathsFromEnviron() 1530 self.prefix = prefix 1531 self.filename = cachePath 1532 self.seconds = seconds 1533 self.mtime = None 1534 self._modmtime = _getMTime(__file__) 1535 1536 self._parser = RegistryParser() 1537 1538 if (READ_CACHE and 1539 os.path.exists(self.filename) and 1540 os.access(self.filename, os.R_OK)): 1541 self.info('Parsing registry: %s', self.filename) 1542 try: 1543 self._parser.parseRegistry(self.filename) 1544 except fxml.ParserError, e: 1545 # this can happen for example if we upgraded to a new version, 1546 # ran, then downgraded again; the registry can then contain 1547 # XML keys that are not understood by this version. 1548 # This is non-fatal, and gets fixed due to a re-scan 1549 self.warning('Could not parse registry %s.', self.filename) 1550 self.debug('fxml.ParserError: %s', log.getExceptionMessage(e)) 1551 1552 self.verify(force=not READ_CACHE)
1553
1554 - def addFile(self, file):
1555 """ 1556 @param file: The file to add, either as an open file object, or 1557 as the name of a file to open. 1558 @type file: str or file. 1559 """ 1560 if isinstance(file, str) and file.endswith('registry.xml'): 1561 self.warning('%s seems to be an old registry in your tree, ' 1562 'please remove it', file) 1563 self.debug('Adding file: %r', file) 1564 self._parser.parseRegistryFile(file)
1565
1566 - def addFromString(self, string):
1567 f = StringIO(string) 1568 self.addFile(f) 1569 f.close()
1570
1571 - def addRegistryPath(self, path, prefix=None):
1572 """ 1573 Add a registry path to this registry, scanning it for registry 1574 snippets. 1575 1576 @param path: a full path containing a PREFIX directory, which will be 1577 scanned for registry files. 1578 @param prefix: directory name under path which will be scanned 1579 (defaults to 'flumotion' and cannot be an empty string). 1580 1581 @rtype: bool 1582 @returns: whether the path could be added 1583 """ 1584 prefix = prefix or self.prefix 1585 self.debug('path %s, prefix %s', path, prefix) 1586 if not os.path.exists(path): 1587 self.warning( 1588 "Cannot add non-existent path '%s' to registry", path) 1589 return False 1590 if not os.path.exists(os.path.join(path, prefix)): 1591 self.warning("Cannot add path '%s' to registry " 1592 "since it does not contain prefix '%s'", path, prefix) 1593 return False 1594 1595 # registry path was either not watched or updated, or a force was 1596 # asked, so reparse 1597 self.info('Scanning registry path %s', path) 1598 registryPath = RegistryDirectory(path, prefix=prefix) 1599 files = registryPath.getFiles() 1600 self.debug('Found %d possible registry files', len(files)) 1601 map(self.addFile, files) 1602 1603 self._parser.addDirectory(registryPath) 1604 return True
1605 1606 # fixme: these methods inconsistenly molest and duplicate those of 1607 # the parser. 1608
1609 - def isEmpty(self):
1610 return len(self._parser._components) == 0
1611
1612 - def getComponent(self, name):
1613 """ 1614 @rtype: L{RegistryEntryComponent} 1615 """ 1616 return self._parser.getComponent(name)
1617
1618 - def hasComponent(self, name):
1619 return name in self._parser._components
1620
1621 - def getComponents(self):
1622 return self._parser.getComponents()
1623
1624 - def getPlug(self, type):
1625 """ 1626 @rtype: L{RegistryEntryPlug} 1627 """ 1628 return self._parser.getPlug(type)
1629
1630 - def hasPlug(self, name):
1631 return name in self._parser._plugs
1632
1633 - def getPlugs(self):
1634 return self._parser.getPlugs()
1635
1636 - def getScenarios(self):
1637 return self._parser.getScenarios()
1638
1639 - def getScenarioByType(self, type):
1640 return self._parser.getScenarioByType(type)
1641
1642 - def getBundles(self):
1643 return self._parser._bundles.values()
1644
1645 - def getDirectories(self):
1646 return self._parser.getDirectories()
1647
1648 - def makeBundlerBasket(self):
1649 """ 1650 @rtype: L{flumotion.common.bundle.BundlerBasket} 1651 """ 1652 1653 def load(): 1654 ret = BundlerBasket(self.mtime) 1655 for b in self.getBundles(): 1656 bundleName = b.getName() 1657 self.debug('Adding bundle %s', bundleName) 1658 for d in b.getDirectories(): 1659 directory = d.getName() 1660 for bundleFilename in d.getFiles(): 1661 try: 1662 basedir = b.getBaseDir() 1663 except errors.NoProjectError, e: 1664 self.warning("Could not load project %s", e.args) 1665 raise 1666 fullpath = os.path.join(basedir, directory, 1667 bundleFilename.getLocation()) 1668 relative = bundleFilename.getRelative() 1669 self.log('Adding path %s as %s to bundle %s', 1670 fullpath, relative, bundleName) 1671 try: 1672 ret.add(bundleName, fullpath, relative) 1673 except Exception, e: 1674 self.debug("Reason: %r", e) 1675 raise RuntimeError( 1676 'Could not add %s to bundle %s (%s)' 1677 % (fullpath, bundleName, e)) 1678 for d in b.getDependencies(): 1679 self.log('Adding dependency of %s on %s', bundleName, d) 1680 ret.depend(bundleName, d) 1681 return ret
1682 1683 try: 1684 return load() 1685 except Exception, e: 1686 self.debug("Could not register bundles the first time: %s", 1687 log.getExceptionMessage(e)) 1688 self.warning("Bundle problem, rebuilding registry") 1689 self.verify(force=True) 1690 try: 1691 return load() 1692 except Exception, e: 1693 self.debug("Could not register bundles the second time: %s", 1694 log.getExceptionMessage(e)) 1695 self.error("Could not not register bundles (%s)", 1696 log.getExceptionMessage(e))
1697
1698 - def dump(self, fd):
1699 """ 1700 Dump the cache of components to the given opened file descriptor. 1701 1702 @type fd: integer 1703 @param fd: open file descriptor to write to 1704 """ 1705 writer = RegistryWriter(self.getComponents(), self.getPlugs(), 1706 self.getBundles(), self.getDirectories()) 1707 writer.dump(fd)
1708
1709 - def clean(self):
1710 """ 1711 Clean the cache of components. 1712 """ 1713 self._parser.clean()
1714
1715 - def rebuildNeeded(self):
1716 if self.mtime is None: 1717 self.log("Rebuild needed: missing mtime") 1718 return True 1719 if not os.path.exists(self.filename): 1720 self.log("Rebuild needed: registry file %s doesn't exists", 1721 self.filename) 1722 return True 1723 1724 # A bit complicated because we want to allow FLU_PROJECT_PATH to 1725 # point to nonexistent directories 1726 registryPaths = python.set(self._paths) 1727 oldRegistryPaths = python.set([directory.getPath() 1728 for directory in self.getDirectories()]) 1729 if registryPaths != oldRegistryPaths: 1730 if oldRegistryPaths - registryPaths: 1731 self.log("Rebuild needed: registry paths removed") 1732 return True 1733 f = filter(os.path.exists, registryPaths - oldRegistryPaths) 1734 if f: 1735 self.log("Rebuild needed: a newly added registry path doesn't " 1736 "exists: %s", f) 1737 return True 1738 1739 registry_modified = self.mtime 1740 for d in self._parser.getDirectories(): 1741 if d.rebuildNeeded(registry_modified): 1742 return True 1743 1744 return False
1745
1746 - def save(self, force=False):
1747 if not force and not self.rebuildNeeded(): 1748 return 1749 1750 self.info('Saving registry to %s', self.filename) 1751 1752 # create parent directory 1753 directory = os.path.split(self.filename)[0] 1754 if not os.path.exists(directory): 1755 try: 1756 makedirs(directory) 1757 except OSError, e: 1758 if e.errno == errno.EACCES: 1759 self.error('Registry directory %s could not be created !' % 1760 directory) 1761 else: 1762 raise 1763 1764 if not os.path.isdir(directory): 1765 self.error('Registry directory %s is not a directory !') 1766 try: 1767 # According to doc http://docs.python.org/library/os.html#os.rename 1768 # If successful, the renaming will be an atomic operation (this is 1769 # a POSIX requirement). 1770 tmp = tempfile.mktemp(dir=directory) 1771 fd = open(tmp, 'w') 1772 self.dump(fd) 1773 os.rename(tmp, self.filename) 1774 except IOError, e: 1775 if e.errno == errno.EACCES: 1776 self.error('Registry file %s could not be created !' % 1777 self.filename) 1778 else: 1779 raise
1780
1781 - def _getRegistryPathsFromEnviron(self):
1782 registryPaths = [configure.pythondir, ] 1783 if 'FLU_PROJECT_PATH' in os.environ: 1784 paths = os.environ['FLU_PROJECT_PATH'] 1785 registryPaths += paths.split(':') 1786 return registryPaths
1787
1788 - def verify(self, force=False):
1789 """ 1790 Verify if the registry is uptodate and rebuild if it is not. 1791 1792 @param force: True if the registry needs rebuilding for sure. 1793 """ 1794 # construct a list of all paths to scan for registry .xml files 1795 if force or self.rebuildNeeded(): 1796 self.info("Rebuilding registry") 1797 if force: 1798 self.info("Rebuild of registry is forced") 1799 if self.rebuildNeeded(): 1800 self.info("Rebuild of registry is needed") 1801 self.clean() 1802 mtime = self.seconds() 1803 for path in self._paths: 1804 if not self.addRegistryPath(path): 1805 self._parser.removeDirectoryByPath(path) 1806 self.mtime = mtime 1807 self.save(True)
1808
1809 - def isUptodate(self):
1810 return self._modmtime >= _getMTime(__file__)
1811 1812
1813 -class RegistrySubsetWriter(RegistryWriter):
1814
1815 - def __init__(self, fromRegistry=None, onlyBundles=None):
1816 """ 1817 @param fromRegistry: The registry to subset, or the default. 1818 @type fromRegistry: L{ComponentRegistry} 1819 @param onlyBundles: If given, only include the subset of the 1820 registry that is provided by bundles whose names are in this 1821 list. 1822 @type onlyBundles: list of str 1823 """ 1824 self.fromRegistry = fromRegistry 1825 self.onlyBundles = onlyBundles
1826
1827 - def dump(self, fd):
1828 reg = self.fromRegistry or getRegistry() 1829 pred = None 1830 bundles = reg.getBundles() 1831 if self.onlyBundles is not None: 1832 bundles = [b for b in bundles 1833 if b.name in self.onlyBundles] 1834 1835 bundledfiles = {} 1836 for b in bundles: 1837 for d in b.getDirectories(): 1838 for f in d.getFiles(): 1839 filename = os.path.join(d.getName(), f.getLocation()) 1840 bundledfiles[filename] = b 1841 1842 def fileIsBundled(basedir, filename): 1843 return os.path.join(basedir, filename) in bundledfiles
1844 1845 pred = lambda c: (filter(lambda f: fileIsBundled(c.getBase(), 1846 f.getFilename()), 1847 c.getFiles()) 1848 or filter(lambda e: fileIsBundled(c.getBase(), 1849 e.getLocation()), 1850 c.getEntries())) 1851 components = filter(pred, reg.getComponents()) 1852 1853 pred = lambda p: p.getEntry().getLocation() in bundledfiles 1854 plugs = filter(pred, reg.getPlugs()) 1855 1856 directories = [] # no need for this 1857 1858 regwriter = RegistryWriter(components, plugs, bundles, directories) 1859 regwriter.dump(fd)
1860 1861 __registry = None 1862 1863
1864 -def makeBundleFromLoadedModules(outfile, outreg, *prefixes):
1865 """ 1866 Make a bundle from a subset of all loaded modules, also writing out 1867 a registry file that can apply to that subset of the global 1868 registry. Suitable for use as a FLU_ATEXIT handler. 1869 1870 @param outfile: The path to which a zip file will be written. 1871 @type outfile: str 1872 @param outreg: The path to which a registry file will be written. 1873 @type outreg: str 1874 @param prefixes: A list of prefixes to which to limit the export. If 1875 not given, package up all modules. For example, "flumotion" would 1876 limit the output to modules that start with "flumotion". 1877 @type prefixes: list of str 1878 """ 1879 from twisted.python import reflect 1880 1881 def getUsedModules(prefixes): 1882 ret = {} 1883 for modname in sys.modules: 1884 if prefixes and not filter(modname.startswith, prefixes): 1885 continue 1886 try: 1887 module = reflect.namedModule(modname) 1888 if hasattr(module, '__file__'): 1889 ret[modname] = module 1890 else: 1891 log.info('makebundle', 'Module %s has no file', module) 1892 except ImportError: 1893 log.info('makebundle', 'Could not import %s', modname) 1894 return ret
1895 1896 def calculateModuleBundleMap(): 1897 allbundles = getRegistry().getBundles() 1898 ret = {} 1899 for bundle in allbundles: 1900 for directory in bundle.getDirectories(): 1901 for bundleFile in directory.getFiles(): 1902 path = os.path.join(directory.getName(), 1903 bundleFile.getLocation()) 1904 parts = path.split(os.path.sep) 1905 if parts[-1].startswith('__init__.py'): 1906 parts.pop() 1907 elif parts[-1].endswith('.py'): 1908 parts[-1] = parts[-1][:-3] 1909 else: 1910 # not a bundled module 1911 continue 1912 modname = '.'.join(parts) 1913 ret[modname] = bundle 1914 return ret 1915 1916 def makeMergedBundler(modules, modulebundlemap): 1917 ret = MergedBundler() 1918 basket = getRegistry().makeBundlerBasket() 1919 for modname in modules: 1920 modfilename = modules[modname].__file__ 1921 if modname in modulebundlemap: 1922 bundleName = modulebundlemap[modname].getName() 1923 for depBundleName in basket.getDependencies(bundleName): 1924 ret.addBundler(basket.getBundlerByName(depBundleName)) 1925 else: 1926 if modfilename.endswith('.pyc'): 1927 modfilename = modfilename[:-1] 1928 if os.path.isdir(modfilename): 1929 with_init = os.path.join(modfilename, '__init__.py') 1930 if os.path.exists(with_init): 1931 modfilename = with_init 1932 nparts = len(modname.split('.')) 1933 if '__init__' in modfilename: 1934 nparts += 1 1935 relpath = os.path.join(*modfilename.split( 1936 os.path.sep)[-nparts:]) 1937 ret.add(modfilename, relpath) 1938 return ret 1939 1940 modules = getUsedModules(prefixes) 1941 modulebundlemap = calculateModuleBundleMap() 1942 bundler = makeMergedBundler(modules, modulebundlemap) 1943 1944 print 'Writing bundle to', outfile 1945 open(outfile, 'w').write(bundler.bundle().getZip()) 1946 1947 print 'Writing registry to', outreg 1948 bundlers_used = [b.name for b in bundler.getSubBundlers()] 1949 regwriter = RegistrySubsetWriter(onlyBundles=bundlers_used) 1950 regwriter.dump(open(outreg, 'w')) 1951 1952
1953 -def getRegistry():
1954 """ 1955 Return the registry. Only one registry will ever be created. 1956 1957 @rtype: L{ComponentRegistry} 1958 """ 1959 global __registry 1960 1961 if not __registry: 1962 log.debug('registry', 'instantiating registry') 1963 __registry = ComponentRegistry() 1964 elif not __registry.isUptodate(): 1965 # When a new version of flumotion gets installed, running managers will 1966 # reread the xml files. Reloading the registry module is required to 1967 # avoid inconsistencies. 1968 log.debug('registry', 'registry module updated, reloading') 1969 reload(sys.modules[__registry.__module__]) 1970 __registry = ComponentRegistry() 1971 1972 return __registry
1973