1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 """parsing of registry, which holds component and bundle information
19 """
20
21 import os
22 import stat
23 import errno
24 import sys
25 import tempfile
26 from StringIO import StringIO
27
28 from xml.sax import saxutils
29 from twisted.spread import pb
30 from twisted.python import runtime
31
32 from flumotion.common import common, log, errors, fxml, python
33 from flumotion.common.python import makedirs
34 from flumotion.common.bundle import BundlerBasket, MergedBundler
35 from flumotion.configure import configure
36
37 __all__ = ['ComponentRegistry', 'registry']
38 __version__ = "$Rev$"
39
40
41
42 READ_CACHE = False
43
44 FLU_RANK_NONE = 0
45
46 _VALID_WIZARD_COMPONENT_TYPES = [
47 'audio-producer',
48 'video-producer',
49 'muxer',
50 'audio-encoder',
51 'video-encoder',
52 'consumer',
53 ]
54
55 _VALID_WIZARD_PLUG_TYPES = [
56 'http-consumer',
57 'httpserver-plug',
58 ]
59
60
62 return os.stat(file)[stat.ST_MTIME]
63
64
65 -class RegistryEntryScenario(pb.Copyable, pb.RemoteCopy):
66 """
67 I represent a <scenario> entry in the registry
68 """
69
70 - def __init__(self, type, description, base, entries):
71 """
72 @param type: the type of this scenario
73 @type type: str
74 @param description: description of this scenario
75 @type description: str
76 @param base: base directory where this scenario is placed
77 @type base: str
78 @param entries: dict of entry point type -> entry
79 @type entries: dict of str -> L{RegistryEntryEntry}
80 """
81 self.type = type
82
83 self.description = description or ""
84 self.base = base
85 self.entries = entries
86
87 - def getEntries(self):
88 """
89 Get the entries asociated with this scenario
90
91 @rtype: list of L{RegistryEntryEntry}
92 """
93 return self.entries.values()
94
95 - def getEntryByType(self, type):
96 """
97 Get the entry point for the given type of entry.
98
99 @param type: The type of the wanted entry.
100 @type type: string
101
102 @rtype: L{RegistryEntryEntry}
103 """
104 return self.entries[type]
105
108
111
112 - def getDescription(self):
113 return self.description
114
115 pb.setUnjellyableForClass(RegistryEntryScenario, RegistryEntryScenario)
116
117
118 -class RegistryEntryComponent(pb.Copyable, pb.RemoteCopy):
119 """
120 I represent a <component> entry in the registry
121 """
122
123
124 __pychecker__ = 'maxargs=15'
125
126 - def __init__(self, filename, type,
127 source, description, base, properties, files,
128 entries, eaters, feeders, needs_sync, clock_priority,
129 sockets, wizards):
130 """
131 @param filename: name of the XML file this component is parsed from
132 @type filename: str
133 @param properties: dict of name -> property
134 @type properties: dict of str -> L{RegistryEntryProperty}
135 @param files: list of files
136 @type files: list of L{RegistryEntryFile}
137 @param entries: dict of entry point type -> entry
138 @type entries: dict of str -> L{RegistryEntryEntry}
139 @param sockets: list of sockets supported by the component
140 @type sockets: list of str
141 @param wizards: list of wizard entries
142 @type wizards: list of L{RegistryEntryWizard}
143 """
144 self.filename = filename
145 self.type = type
146 self.source = source
147 self.description = description
148
149 if not self.description:
150 self.description = ""
151 self.base = base
152 self.properties = properties
153 self.files = files
154 self.entries = entries
155 self.eaters = eaters
156 self.feeders = feeders
157 self.needs_sync = needs_sync
158 self.clock_priority = clock_priority
159 self.sockets = sockets
160 self.wizards = wizards
161
162 - def getProperties(self):
163 """
164 Get a list of all properties.
165
166 @rtype: list of L{RegistryEntryProperty}
167 """
168 return self.properties.values()
169
170 - def hasProperty(self, name):
171 """
172 Check if the component has a property with the given name.
173 """
174 return name in self.properties.keys()
175
176 - def getFiles(self):
177 """
178 @rtype: list of L{RegistryEntryFile}
179 """
180 return self.files
181
182 - def getEntries(self):
183 return self.entries.values()
184
185 - def getEntryByType(self, type):
186 """
187 Get the entry point for the given type of entry.
188
189 @type type: string
190 """
191 return self.entries[type]
192
193 - def getGUIEntry(self):
194 if not self.files:
195 return
196
197
198 if len(self.files) > 1:
199 return
200
201 return self.files[0].getFilename()
202
205
208
209 - def getDescription(self):
210 return self.description
211
212 - def getSource(self):
214
215 - def getEaters(self):
217
218 - def getFeeders(self):
220
222 return self.needs_sync
223
225 return self.clock_priority
226
227 - def getSockets(self):
229 pb.setUnjellyableForClass(RegistryEntryComponent, RegistryEntryComponent)
230
231
233 """
234 I represent a <plug> entry in the registry
235 """
236
237 - def __init__(self, filename, type,
238 description, socket, entries, properties, wizards):
239 """
240 @param filename: name of the XML file this plug is parsed from
241 @type filename: str
242 @param type: the type of plug
243 @type type: str
244 @param description: the translatable description of the plug
245 @type description: str
246 @param socket: the fully qualified class name of the socket this
247 plug can be plugged in to
248 @type socket: str
249 @param entries: entry points for instantiating the plug
250 @type entries: list of L{RegistryEntryEntry}
251 @param properties: properties of the plug
252 @type properties: dict of str -> L{RegistryEntryProperty}
253 @param wizards: list of wizard entries
254 @type wizards: list of L{RegistryEntryWizard}
255 """
256 self.filename = filename
257 self.type = type
258 self.description = description
259 self.socket = socket
260 self.entries = entries
261 self.properties = properties
262 self.wizards = wizards
263
264 - def getProperties(self):
265 """
266 Get a list of all properties.
267
268 @rtype: list of L{RegistryEntryProperty}
269 """
270 return self.properties.values()
271
272 - def hasProperty(self, name):
273 """
274 Check if the component has a property with the given name.
275 """
276 return name in self.properties.keys()
277
278 - def getEntryByType(self, type):
279 """
280 Get the entry point for the given type of entry.
281
282 @type type: string
283 """
284 return self.entries[type]
285
286 - def getEntry(self):
287 return self.entries['default']
288
289 - def getEntries(self):
290 return self.entries.values()
291
294
295 - def getDescription(self):
296 return self.description
297
298 - def getSocket(self):
300
301
303 "This class represents a <bundle> entry in the registry"
304
305 - def __init__(self, name, project, under, dependencies, directories):
306 self.name = name
307 self.project = project
308 self.under = under
309 self.dependencies = dependencies
310 self.directories = directories
311
312 - def __repr__(self):
313 return '<Bundle name=%s>' % self.name
314
317
318 - def getDependencies(self):
319 """
320 @rtype: list of str
321 """
322 return self.dependencies
323
324 - def getDirectories(self):
325 """
326 @rtype: list of L{RegistryEntryBundleDirectory}
327 """
328 return self.directories
329
330 - def getProject(self):
332
333 - def getUnder(self):
335
336 - def getBaseDir(self):
337 if self.project == configure.PACKAGE:
338 return getattr(configure, self.under)
339
340 from flumotion.project import project
341 return project.get(self.project, self.under)
342
343
345 "This class represents a <directory> entry in the registry"
346
347 - def __init__(self, name, files):
348 self.name = name
349 self.files = files
350
353
354 - def getFiles(self):
356
357
359 "This class represents a <filename> entry in the registry"
360
361 - def __init__(self, location, relative):
362 self.location = location
363 self.relative = relative
364
365 - def getLocation(self):
367
368 - def getRelative(self):
370
371
373 "This class represents a <property> entry in the registry"
374
375 - def __init__(self, name, type, description,
376 required=False, multiple=False):
377 self.name = name
378 self.type = type
379 self.description = description
380
381 if not self.description:
382 self.description = ""
383 self.required = required
384 self.multiple = multiple
385
386 - def __repr__(self):
387 return '<Property name=%s>' % self.name
388
391
394
395 - def getDescription(self):
396 return self.description
397
398 - def isRequired(self):
400
401 - def isMultiple(self):
403
404
405 -class RegistryEntryCompoundProperty(RegistryEntryProperty):
406 "This class represents a <compound-property> entry in the registry"
407
408 - def __init__(self, name, description, properties, required=False,
409 multiple=False):
413
414 - def __repr__(self):
415 return '<Compound-property name=%s>' % self.name
416
417 - def getProperties(self):
418 """
419 Get a list of all sub-properties.
420
421 @rtype: list of L{RegistryEntryProperty}
422 """
423 return self.properties.values()
424
425 - def hasProperty(self, name):
426 """
427 Check if the compound-property has a sub-property with the
428 given name.
429 """
430 return name in self.properties
431
432
434 "This class represents a <file> entry in the registry"
435
436 - def __init__(self, filename, type):
437 self.filename = filename
438 self.type = type
439
441 return os.path.basename(self.filename)
442
445
446 - def getFilename(self):
448
449 - def isType(self, type):
450 return self.type == type
451
452
454 "This class represents a <entry> entry in the registry"
455
456 - def __init__(self, type, location, function):
457 self.type = type
458 self.location = location
459 self.function = function
460
463
464 - def getLocation(self):
466
467 - def getModuleName(self, base=None):
468 if base:
469 path = os.path.join(base, self.getLocation())
470 else:
471 path = self.getLocation()
472 return common.pathToModuleName(path)
473
474 - def getFunction(self):
476
477
479 "This class represents a <eater> entry in the registry"
480
481 - def __init__(self, name, required=True, multiple=False):
482 self.name = name
483 self.required = required
484 self.multiple = multiple
485
488
489 - def getRequired(self):
491
492 - def getMultiple(self):
494
495
496 -class RegistryEntryWizard(pb.Copyable):
497 "This class represents a <wizard> entry in the registry"
498
499 - def __init__(self, componentType, type, description, feeder,
500 eater, accepts, provides, rank=FLU_RANK_NONE):
501 self.componentType = componentType
502 self.type = type
503 self.description = description
504 self.feeder = feeder
505 self.eater = eater
506 self.accepts = accepts
507 self.provides = provides
508 self.rank = rank
509
510 - def __repr__(self):
511 return '<wizard %s type=%s, feeder=%s>' % (self.componentType,
512 self.type, self.feeder)
513
514
516 """
517 This class represents an <accept-format> or <provide-format>
518 entry in the registry
519 """
520
522 self.media_type = media_type
523
524
526 """
527 Registry parser
528
529 I have two modes, one to parse registries and another one to parse
530 standalone component files.
531
532 For parsing registries use the parseRegistry function and for components
533 use parseRegistryFile.
534
535 I also have a list of all components and directories which the
536 registry uses (instead of saving its own copy)
537 """
538
541
543 self._components = {}
544 self._directories = {}
545 self._bundles = {}
546 self._plugs = {}
547 self._scenarios = {}
548
550 return self._components.values()
551
558
560 return self._scenarios.values()
561
563 if type in self._scenarios:
564 return self._scenarios[type]
565 return None
566
568 return self._plugs.values()
569
576
578
579
580
581
582 components = {}
583
584 def addComponent(comp):
585 components[comp.getType()] = comp
586
587 parsers = {'component': (self._parseComponent, addComponent)}
588 self.parseFromTable(node, parsers)
589
590 return components
591
593
594
595
596
597
598
599
600
601
602
603
604
605 componentType, baseDir, description, _description = \
606 self.parseAttributes(node,
607 required=('type', 'base'),
608 optional=('description', '_description'))
609
610
611 if description:
612 import warnings
613 warnings.warn(
614 "Please change '<component description=...'"
615 " to '<component _description=...' for %s" % componentType,
616 DeprecationWarning)
617 if _description:
618 description = _description
619
620 files = []
621 source = fxml.Box(None)
622 entries = {}
623 eaters = []
624 feeders = []
625 synchronization = fxml.Box((False, 100))
626 sockets = []
627 properties = {}
628 wizards = []
629
630
631
632
633
634
635
636
637 parsers = {
638 'source': (self._parseSource, source.set),
639 'properties': (self._parseProperties, properties.update),
640 'files': (self._parseFiles, files.extend),
641 'entries': (self._parseEntries, entries.update),
642 'eater': (self._parseEater, eaters.append),
643 'feeder': (self._parseFeeder, feeders.append),
644 'synchronization': (self._parseSynchronization,
645 synchronization.set),
646 'sockets': (self._parseSockets, sockets.extend),
647 'wizard': (self._parseComponentWizard, wizards.append),
648 }
649 self.parseFromTable(node, parsers)
650
651 source = source.unbox()
652 needs_sync, clock_priority = synchronization.unbox()
653
654 return RegistryEntryComponent(self.filename,
655 componentType, source, description,
656 baseDir, properties, files,
657 entries, eaters, feeders,
658 needs_sync, clock_priority,
659 sockets, wizards)
660
662
663
664
665
666 scenarios = {}
667
668 def addScenario(scenario):
669 scenarios[scenario.getType()] = scenario
670
671 parsers = {'scenario': (self._parseScenario, addScenario)}
672 self.parseFromTable(node, parsers)
673
674 return scenarios
675
677
678
679
680
681 scenarioType, baseDir, description = \
682 self.parseAttributes(node,
683 required=('type', 'base'),
684 optional=('_description', ))
685
686 entries = {}
687
688 parsers = {
689 'entries': (self._parseEntries, entries.update),
690 }
691
692 self.parseFromTable(node, parsers)
693
694 return RegistryEntryScenario(scenarioType, description,
695 baseDir, entries)
696
701
703
704
705
706
707 attrs = self.parseAttributes(node, required=('name', 'type'),
708 optional=('required', 'multiple', 'description', '_description'))
709 name, propertyType, required, multiple, description, _d = attrs
710 if description:
711 import warnings
712 warnings.warn("Please change '<property description=...'"
713 " to '<property _description=...' for %s" % name,
714 DeprecationWarning)
715 if _d:
716 description = _d
717
718
719 allowed = ('string', 'rawstring', 'int', 'long', 'bool',
720 'float', 'fraction')
721 if propertyType not in allowed:
722 raise fxml.ParserError(
723 "<property> %s's type is not one of %s" % (
724 name, ", ".join(allowed)))
725 required = common.strToBool(required)
726 multiple = common.strToBool(multiple)
727 return RegistryEntryProperty(name, propertyType, description,
728 required=required, multiple=multiple)
729
731
732
733
734
735
736
737
738 attrs = self.parseAttributes(node, required=('name', ),
739 optional=('required', 'multiple', 'description', '_description'))
740 name, required, multiple, description, _description = attrs
741 if description:
742 import warnings
743 warnings.warn("Please change '<compound-property description=...'"
744 " to '<compound-property _description=...' for %s" % name,
745 DeprecationWarning)
746 if _description:
747 description = _description
748
749
750 required = common.strToBool(required)
751 multiple = common.strToBool(multiple)
752
753 properties = {}
754
755 def addProperty(prop):
756 properties[prop.getName()] = prop
757
758 parsers = {'property': (self._parseProperty, addProperty),
759 'compound-property': (self._parseCompoundProperty,
760 addProperty)}
761 self.parseFromTable(node, parsers)
762
763 return RegistryEntryCompoundProperty(name, description, properties,
764 required=required, multiple=multiple)
765
776
777 parsers = {'property': (self._parseProperty, addProperty),
778 'compound-property': (self._parseCompoundProperty,
779 addProperty)}
780
781 self.parseFromTable(node, parsers)
782
783 return properties
784
793
805
812
824
825 - def _parseEntry(self, node):
826 attrs = self.parseAttributes(node, ('type', 'location', 'function'))
827 entryType, location, function = attrs
828 return RegistryEntryEntry(entryType, location, function)
829
831
832
833
834
835
836 entries = {}
837
838 def addEntry(entry):
839 if entry.getType() in entries:
840 raise fxml.ParserError("entry %s already specified"
841 % entry.getType())
842 entries[entry.getType()] = entry
843
844 parsers = {'entry': (self._parseEntry, addEntry)}
845
846 self.parseFromTable(node, parsers)
847
848 return entries
849
860
865
873
874 - def _parsePlugEntry(self, node):
875 attrs = self.parseAttributes(node,
876 ('location', 'function'), ('type', ))
877 location, function, entryType = attrs
878 if not entryType:
879 entryType = 'default'
880 return RegistryEntryEntry(entryType, location, function)
881
883 return {'default': self._parsePlugEntry(node)}
884
886
887
888
889
890
891 entries = {}
892
893 def addEntry(entry):
894 if entry.getType() in entries:
895 raise fxml.ParserError("entry %s already specified"
896 % entry.getType())
897 entries[entry.getType()] = entry
898
899 parsers = {'entry': (self._parsePlugEntry, addEntry)}
900
901 self.parseFromTable(node, parsers)
902
903 return entries
904
906
907
908
909
910
911
912
913
914 plugType, socket, description = \
915 self.parseAttributes(node, required=('type', 'socket'),
916 optional=('_description', ))
917
918 if not description:
919 import warnings
920 warnings.warn(
921 "Please add '_description=...' attribute to plug '%s'" %
922 plugType,
923 DeprecationWarning)
924 description = 'TODO'
925
926 entries = {}
927 properties = {}
928 wizards = []
929
930 parsers = {
931 'entries': (self._parsePlugEntries, entries.update),
932
933 'entry': (self._parseDefaultPlugEntry, entries.update),
934 'properties': (self._parseProperties, properties.update),
935 'wizard': (self._parsePlugWizard, wizards.append),
936 }
937
938 self.parseFromTable(node, parsers)
939
940 if not 'default' in entries:
941 raise fxml.ParserError(
942 "<plug> %s needs a default <entry>" % plugType)
943
944 return RegistryEntryPlug(self.filename, plugType, description,
945 socket, entries, properties,
946 wizards)
947
959
960 parsers = {'plug': (self._parsePlug, addPlug)}
961 self.parseFromTable(node, parsers)
962
963 return plugs
964
965
966
968 """
969 @param file: The file to parse, either as an open file object,
970 or as the name of a file to open.
971 @type file: str or file.
972 """
973 if isinstance(file, basestring):
974 self.filename = file
975 else:
976 self.filename = getattr(file, 'name', '<string>')
977 root = self.getRoot(file)
978 node = root.documentElement
979
980 if node.nodeName != 'registry':
981
982
983 self.debug('%s does not have registry as root tag', self.filename)
984 return
985
986
987 self._parseRoot(node, disallowed=['directories'])
988 root.unlink()
989
999
1000 parsers = {'bundle': (self._parseBundle, addBundle)}
1001 self.parseFromTable(node, parsers)
1002
1003 return bundles
1004
1006
1007
1008
1009
1010
1011 attrs = self.parseAttributes(node, ('name', ), ('project', 'under'))
1012 name, project, under = attrs
1013 project = project or configure.PACKAGE
1014 under = under or 'pythondir'
1015
1016 dependencies = []
1017 directories = []
1018
1019 parsers = {'dependencies': (self._parseBundleDependencies,
1020 dependencies.extend),
1021 'directories': (self._parseBundleDirectories,
1022 directories.extend)}
1023 self.parseFromTable(node, parsers)
1024
1025 return RegistryEntryBundle(name, project, under,
1026 dependencies, directories)
1027
1031
1043
1055
1064
1075
1076 parsers = {'filename': (parseFilename, filenames.append)}
1077 self.parseFromTable(node, parsers)
1078
1079 return RegistryEntryBundleDirectory(name, filenames)
1080
1081
1082
1084 """
1085 @param file: The file to parse, either as an open file object,
1086 or as the name of a file to open.
1087 @type file: str or file.
1088 """
1089 if isinstance(file, basestring):
1090 self.filename = file
1091 else:
1092 self.filename = getattr(file, 'name', '<string>')
1093 root = self.getRoot(file)
1094 self._parseRoot(root.documentElement)
1095 root.unlink()
1096
1098 return self._directories.values()
1099
1101 return self._directories[name]
1102
1104 """
1105 Add a registry path object to the parser.
1106
1107 @type directory: {RegistryDirectory}
1108 """
1109 self._directories[directory.getPath()] = directory
1110
1112 """
1113 Remove a directory from the parser given the path.
1114 Used when the path does not actually contain any registry information.
1115 """
1116 if path in self._directories.keys():
1117 del self._directories[path]
1118
1120
1121
1122
1123
1124
1125 parsers = {'components': (self._parseComponents,
1126 self._components.update),
1127 'directories': (self._parseDirectories,
1128 self._directories.update),
1129 'bundles': (self._parseBundles, self._bundles.update),
1130 'plugs': (self._parsePlugs, self._plugs.update),
1131 'scenarios': (self._parseScenarios, self._scenarios.update)}
1132
1133 if disallowed:
1134 for k in disallowed:
1135 del parsers[k]
1136
1137 self.parseFromTable(node, parsers)
1138
1140
1141
1142
1143
1144 directories = {}
1145
1146 def addDirectory(d):
1147 directories[d.getPath()] = d
1148
1149 parsers = {'directory': (self._parseDirectory, addDirectory)}
1150 self.parseFromTable(node, parsers)
1151
1152 return directories
1153
1158
1161
1164
1166
1167
1168
1169
1170
1171
1172 attrs = self.parseAttributes(node,
1173 ('type', '_description'),
1174 ('feeder', 'eater', 'rank'))
1175 wizardType, description, feeder, eater, rank = attrs
1176
1177 accepts = []
1178 provides = []
1179 parsers = {
1180 'accept-format': (self._parseAcceptFormat,
1181 lambda n: accepts.append(n)),
1182 'provide-format': (self._parseProvideFormat,
1183 lambda n: provides.append(n)),
1184 }
1185 self.parseFromTable(node, parsers)
1186
1187 parent_type = node.parentNode.getAttribute('type')
1188
1189 if not wizardType in validTypes:
1190 raise fxml.ParserError(
1191 "<wizard>'s type attribute is %s must be one of %s" % (
1192 parent_type,
1193 ', '.join(validTypes)))
1194 rank = int(rank or FLU_RANK_NONE)
1195 isProducer = wizardType.endswith('-producer')
1196 isEncoder = wizardType.endswith('-encoder')
1197 isMuxer = (wizardType == 'muxer')
1198 isConsumer = wizardType.endswith('-consumer')
1199
1200 err = None
1201
1202 if accepts and (isProducer or isEncoder):
1203 err = ('<wizard type="%s"> does not allow an accepted '
1204 'media-type.') % (parent_type, )
1205
1206 elif not accepts and (isMuxer or isConsumer):
1207 err = ('<wizard type="%s"> requires at least one accepted '
1208 'media-type.') % (parent_type, )
1209
1210 elif provides and (isProducer or isConsumer):
1211 err = ('<wizard type="%s"> does not allow a provided '
1212 'media-type.') % (parent_type, )
1213
1214 if len(provides) != 1 and (isEncoder or isMuxer):
1215 err = ('<wizard type="%s"> requires exactly one provided '
1216 'media-type.') % (parent_type, )
1217
1218 if err:
1219 raise fxml.ParserError(err)
1220
1221 return RegistryEntryWizard(parent_type, wizardType, description,
1222 feeder, eater, accepts, provides, rank)
1223
1228
1233
1234
1235
1236
1237
1239 """
1240 I represent a directory under a path managed by the registry.
1241 I can be queried for a list of partial registry .xml files underneath
1242 the given path, under the given prefix.
1243 """
1244
1250
1252 return "<RegistryDirectory %s>" % self._path
1253
1255 """
1256 Get all files ending in .xml from all directories under the given root.
1257
1258 @type root: string
1259 @param root: the root directory under which to search
1260
1261 @returns: a list of .xml files, relative to the given root directory
1262 """
1263 files = []
1264 dirs = []
1265
1266 if os.path.exists(root):
1267 try:
1268 directory_files = os.listdir(root)
1269 except OSError, e:
1270 if e.errno == errno.EACCES:
1271 return files, dirs
1272 else:
1273 raise
1274
1275 dirs.append(root)
1276
1277 for entry in directory_files:
1278 path = os.path.join(root, entry)
1279
1280 if not os.path.isdir(path):
1281 if path.endswith('.xml'):
1282 files.append(path)
1283
1284
1285 elif entry != '.svn':
1286 newFiles, newDirs = self._getFileLists(path)
1287 files.extend(newFiles)
1288 dirs.extend(newDirs)
1289
1290 return files, dirs
1291
1293
1294 def _rebuildNeeded(f):
1295 try:
1296 if _getMTime(f) > mtime:
1297 self.debug("Path %s changed since registry last "
1298 "scanned", f)
1299 return True
1300 return False
1301 except OSError:
1302 self.debug("Failed to stat file %s, need to rescan", f)
1303 return True
1304
1305 for f in self._files:
1306 if _rebuildNeeded(f):
1307 return True
1308 for f in self._dirs:
1309 if _rebuildNeeded(f):
1310 return True
1311 return False
1312
1314 """
1315 Return a list of all .xml registry files underneath this registry
1316 path.
1317 """
1318 return self._files
1319
1322
1323
1325
1326 - def __init__(self, components, plugs, bundles, directories):
1327 """
1328 @param components: components to write
1329 @type components: list of L{RegistryEntryComponent}
1330 @param plugs: plugs to write
1331 @type plugs: list of L{RegistryEntryPlug}
1332 @param bundles: bundles to write
1333 @type bundles: list of L{RegistryEntryBundle}
1334 @param directories: directories to write
1335 @type directories: list of L{RegistryEntryBundleDirectory}
1336 """
1337 self.components = components
1338 self.plugs = plugs
1339 self.bundles = bundles
1340 self.directories = directories
1341
1342 - def dump(self, fd):
1343 """
1344 Dump the cache of components to the given opened file descriptor.
1345
1346 @type fd: integer
1347 @param fd: open file descriptor to write to
1348 """
1349
1350 def w(i, msg):
1351 print >> fd, ' '*i + msg
1352
1353 def e(attr):
1354 return saxutils.quoteattr(attr)
1355
1356 def _dump_proplist(i, proplist, ioff=2):
1357 for prop in proplist:
1358 if isinstance(prop, RegistryEntryCompoundProperty):
1359 _dump_compound(i, prop)
1360 else:
1361 w(i, ('<property name="%s" type="%s"'
1362 % (prop.getName(), prop.getType())))
1363 w(i, (' _description=%s'
1364 % (e(prop.getDescription()), )))
1365 w(i, (' required="%s" multiple="%s"/>'
1366 % (prop.isRequired(), prop.isMultiple())))
1367
1368 def _dump_compound(i, cprop, ioff=2):
1369 w(i, ('<compound-property name="%s"' % (cprop.getName(), )))
1370 w(i, (' _description=%s'
1371 % (e(cprop.getDescription()), )))
1372 w(i, (' required="%s" multiple="%s">'
1373 % (cprop.isRequired(), cprop.isMultiple())))
1374 _dump_proplist(i + ioff, cprop.getProperties())
1375 w(i, ('</compound-property>'))
1376
1377 def _dump_entries(i, entries):
1378 if not entries:
1379 return
1380
1381 w(i, '<entries>')
1382 for entry in entries:
1383 w(i+2, '<entry type="%s" location="%s" function="%s"/>' % (
1384 entry.getType(),
1385 entry.getLocation(),
1386 entry.getFunction()))
1387 w(i, '</entries>')
1388
1389 w(0, '<registry>')
1390 w(0, '')
1391
1392
1393 w(2, '<components>')
1394 w(0, '')
1395 for component in self.components:
1396 w(4, '<component type="%s" base="%s"' % (
1397 component.getType(), component.getBase()))
1398 w(4, ' _description=%s>'
1399 % (e(component.getDescription()), ))
1400
1401 w(6, '<source location="%s"/>' % component.getSource())
1402 for x in component.getEaters():
1403 w(6, '<eater name="%s" required="%s" multiple="%s"/>'
1404 % (x.getName(), x.getRequired() and "yes" or "no",
1405 x.getMultiple() and "yes" or "no"))
1406 for x in component.getFeeders():
1407 w(6, '<feeder name="%s"/>' % x)
1408 w(6, '<synchronization required="%s" clock-priority="%d"/>'
1409 % (component.getNeedsSynchronization() and "yes" or "no",
1410 component.getClockPriority()))
1411
1412 sockets = component.getSockets()
1413 if sockets:
1414 w(6, '<sockets>')
1415 for socket in sockets:
1416 w(8, '<socket type="%s"/>' % socket)
1417 w(6, '</sockets>')
1418
1419 w(6, '<properties>')
1420 _dump_proplist(8, component.getProperties())
1421 w(6, '</properties>')
1422
1423 for wizard in component.wizards:
1424 rank = ''
1425 if wizard.rank:
1426 rank = ' rank="%d"' % wizard.rank
1427 w(6, '<wizard type="%s" _description="%s" feeder="%s"%s>' % (
1428 wizard.type,
1429 e(wizard.description),
1430 wizard.feeder,
1431 rank))
1432 for accept in wizard.accepts:
1433 w(8, '<accept-format media-type="%s"/>' % (
1434 accept.media_type))
1435 for provide in wizard.provides:
1436 w(8, '<provide-format media-type="%s"/>' % (
1437 provide.media_type))
1438 w(6, '</wizard>')
1439
1440 registryEntryFiles = component.getFiles()
1441 if registryEntryFiles:
1442 w(6, '<files>')
1443 for entryFile in registryEntryFiles:
1444 w(8, '<file name="%s" type="%s"/>' % (
1445 entryFile.getName(),
1446 entryFile.getType()))
1447 w(6, '</files>')
1448
1449 _dump_entries(6, component.getEntries())
1450
1451 w(4, '</component>')
1452 w(0, '')
1453
1454 w(2, '</components>')
1455 w(0, '')
1456
1457
1458 w(2, '<plugs>')
1459 w(0, '')
1460 for plug in self.plugs:
1461 w(4, '<plug type="%s" socket="%s" _description="%s">'
1462 % (plug.getType(), plug.getSocket(), plug.getDescription()))
1463
1464 _dump_entries(6, plug.getEntries())
1465
1466 w(6, '<properties>')
1467 _dump_proplist(8, plug.getProperties())
1468 w(6, '</properties>')
1469
1470 w(4, '</plug>')
1471 w(0, '')
1472
1473 w(2, '</plugs>')
1474 w(0, '')
1475
1476
1477 w(2, '<bundles>')
1478 for bundle in self.bundles:
1479 w(4, '<bundle name="%s" under="%s" project="%s">' % (
1480 bundle.getName(), bundle.getUnder(), bundle.getProject()))
1481
1482 dependencies = bundle.getDependencies()
1483 if dependencies:
1484 w(6, '<dependencies>')
1485 for dependency in dependencies:
1486 w(8, '<dependency name="%s"/>' % dependency)
1487 w(6, '</dependencies>')
1488
1489 bundleDirectories = bundle.getDirectories()
1490 if bundleDirectories:
1491 w(6, '<directories>')
1492 for directory in bundleDirectories:
1493 w(8, '<directory name="%s">' % directory.getName())
1494 for filename in directory.getFiles():
1495 w(10, '<filename location="%s" relative="%s"/>' % (
1496 filename.getLocation(), filename.getRelative()))
1497 w(8, '</directory>')
1498 w(6, '</directories>')
1499
1500 w(4, '</bundle>')
1501 w(0, '')
1502 w(2, '</bundles>')
1503
1504
1505
1506 directories = self.directories
1507 if directories:
1508 w(2, '<directories>')
1509 w(0, '')
1510 for d in directories:
1511 w(4, '<directory filename="%s"/>' % d.getPath())
1512 w(2, '</directories>')
1513 w(0, '')
1514
1515 w(0, '</registry>')
1516
1517
1519 """Registry, this is normally not instantiated."""
1520
1521 logCategory = 'registry'
1522 defaultCachePath = os.path.join(configure.registrydir, 'registry.xml')
1523
1553
1555 """
1556 @param file: The file to add, either as an open file object, or
1557 as the name of a file to open.
1558 @type file: str or file.
1559 """
1560 if isinstance(file, str) and file.endswith('registry.xml'):
1561 self.warning('%s seems to be an old registry in your tree, '
1562 'please remove it', file)
1563 self.debug('Adding file: %r', file)
1564 self._parser.parseRegistryFile(file)
1565
1567 f = StringIO(string)
1568 self.addFile(f)
1569 f.close()
1570
1572 """
1573 Add a registry path to this registry, scanning it for registry
1574 snippets.
1575
1576 @param path: a full path containing a PREFIX directory, which will be
1577 scanned for registry files.
1578 @param prefix: directory name under path which will be scanned
1579 (defaults to 'flumotion' and cannot be an empty string).
1580
1581 @rtype: bool
1582 @returns: whether the path could be added
1583 """
1584 prefix = prefix or self.prefix
1585 self.debug('path %s, prefix %s', path, prefix)
1586 if not os.path.exists(path):
1587 self.warning(
1588 "Cannot add non-existent path '%s' to registry", path)
1589 return False
1590 if not os.path.exists(os.path.join(path, prefix)):
1591 self.warning("Cannot add path '%s' to registry "
1592 "since it does not contain prefix '%s'", path, prefix)
1593 return False
1594
1595
1596
1597 self.info('Scanning registry path %s', path)
1598 registryPath = RegistryDirectory(path, prefix=prefix)
1599 files = registryPath.getFiles()
1600 self.debug('Found %d possible registry files', len(files))
1601 map(self.addFile, files)
1602
1603 self._parser.addDirectory(registryPath)
1604 return True
1605
1606
1607
1608
1610 return len(self._parser._components) == 0
1611
1613 """
1614 @rtype: L{RegistryEntryComponent}
1615 """
1616 return self._parser.getComponent(name)
1617
1619 return name in self._parser._components
1620
1623
1625 """
1626 @rtype: L{RegistryEntryPlug}
1627 """
1628 return self._parser.getPlug(type)
1629
1631 return name in self._parser._plugs
1632
1635
1638
1641
1643 return self._parser._bundles.values()
1644
1647
1649 """
1650 @rtype: L{flumotion.common.bundle.BundlerBasket}
1651 """
1652
1653 def load():
1654 ret = BundlerBasket(self.mtime)
1655 for b in self.getBundles():
1656 bundleName = b.getName()
1657 self.debug('Adding bundle %s', bundleName)
1658 for d in b.getDirectories():
1659 directory = d.getName()
1660 for bundleFilename in d.getFiles():
1661 try:
1662 basedir = b.getBaseDir()
1663 except errors.NoProjectError, e:
1664 self.warning("Could not load project %s", e.args)
1665 raise
1666 fullpath = os.path.join(basedir, directory,
1667 bundleFilename.getLocation())
1668 relative = bundleFilename.getRelative()
1669 self.log('Adding path %s as %s to bundle %s',
1670 fullpath, relative, bundleName)
1671 try:
1672 ret.add(bundleName, fullpath, relative)
1673 except Exception, e:
1674 self.debug("Reason: %r", e)
1675 raise RuntimeError(
1676 'Could not add %s to bundle %s (%s)'
1677 % (fullpath, bundleName, e))
1678 for d in b.getDependencies():
1679 self.log('Adding dependency of %s on %s', bundleName, d)
1680 ret.depend(bundleName, d)
1681 return ret
1682
1683 try:
1684 return load()
1685 except Exception, e:
1686 self.debug("Could not register bundles the first time: %s",
1687 log.getExceptionMessage(e))
1688 self.warning("Bundle problem, rebuilding registry")
1689 self.verify(force=True)
1690 try:
1691 return load()
1692 except Exception, e:
1693 self.debug("Could not register bundles the second time: %s",
1694 log.getExceptionMessage(e))
1695 self.error("Could not not register bundles (%s)",
1696 log.getExceptionMessage(e))
1697
1698 - def dump(self, fd):
1708
1710 """
1711 Clean the cache of components.
1712 """
1713 self._parser.clean()
1714
1716 if self.mtime is None:
1717 self.log("Rebuild needed: missing mtime")
1718 return True
1719 if not os.path.exists(self.filename):
1720 self.log("Rebuild needed: registry file %s doesn't exists",
1721 self.filename)
1722 return True
1723
1724
1725
1726 registryPaths = python.set(self._paths)
1727 oldRegistryPaths = python.set([directory.getPath()
1728 for directory in self.getDirectories()])
1729 if registryPaths != oldRegistryPaths:
1730 if oldRegistryPaths - registryPaths:
1731 self.log("Rebuild needed: registry paths removed")
1732 return True
1733 f = filter(os.path.exists, registryPaths - oldRegistryPaths)
1734 if f:
1735 self.log("Rebuild needed: a newly added registry path doesn't "
1736 "exists: %s", f)
1737 return True
1738
1739 registry_modified = self.mtime
1740 for d in self._parser.getDirectories():
1741 if d.rebuildNeeded(registry_modified):
1742 return True
1743
1744 return False
1745
1746 - def save(self, force=False):
1747 if not force and not self.rebuildNeeded():
1748 return
1749
1750 self.info('Saving registry to %s', self.filename)
1751
1752
1753 directory = os.path.split(self.filename)[0]
1754 if not os.path.exists(directory):
1755 try:
1756 makedirs(directory)
1757 except OSError, e:
1758 if e.errno == errno.EACCES:
1759 self.error('Registry directory %s could not be created !' %
1760 directory)
1761 else:
1762 raise
1763
1764 if not os.path.isdir(directory):
1765 self.error('Registry directory %s is not a directory !')
1766 try:
1767
1768
1769
1770 tmp = tempfile.mktemp(dir=directory)
1771 fd = open(tmp, 'w')
1772 self.dump(fd)
1773 os.rename(tmp, self.filename)
1774 except IOError, e:
1775 if e.errno == errno.EACCES:
1776 self.error('Registry file %s could not be created !' %
1777 self.filename)
1778 else:
1779 raise
1780
1782 registryPaths = [configure.pythondir, ]
1783 if 'FLU_PROJECT_PATH' in os.environ:
1784 paths = os.environ['FLU_PROJECT_PATH']
1785 registryPaths += paths.split(':')
1786 return registryPaths
1787
1788 - def verify(self, force=False):
1789 """
1790 Verify if the registry is uptodate and rebuild if it is not.
1791
1792 @param force: True if the registry needs rebuilding for sure.
1793 """
1794
1795 if force or self.rebuildNeeded():
1796 self.info("Rebuilding registry")
1797 if force:
1798 self.info("Rebuild of registry is forced")
1799 if self.rebuildNeeded():
1800 self.info("Rebuild of registry is needed")
1801 self.clean()
1802 mtime = self.seconds()
1803 for path in self._paths:
1804 if not self.addRegistryPath(path):
1805 self._parser.removeDirectoryByPath(path)
1806 self.mtime = mtime
1807 self.save(True)
1808
1810 return self._modmtime >= _getMTime(__file__)
1811
1812
1814
1815 - def __init__(self, fromRegistry=None, onlyBundles=None):
1816 """
1817 @param fromRegistry: The registry to subset, or the default.
1818 @type fromRegistry: L{ComponentRegistry}
1819 @param onlyBundles: If given, only include the subset of the
1820 registry that is provided by bundles whose names are in this
1821 list.
1822 @type onlyBundles: list of str
1823 """
1824 self.fromRegistry = fromRegistry
1825 self.onlyBundles = onlyBundles
1826
1827 - def dump(self, fd):
1844
1845 pred = lambda c: (filter(lambda f: fileIsBundled(c.getBase(),
1846 f.getFilename()),
1847 c.getFiles())
1848 or filter(lambda e: fileIsBundled(c.getBase(),
1849 e.getLocation()),
1850 c.getEntries()))
1851 components = filter(pred, reg.getComponents())
1852
1853 pred = lambda p: p.getEntry().getLocation() in bundledfiles
1854 plugs = filter(pred, reg.getPlugs())
1855
1856 directories = []
1857
1858 regwriter = RegistryWriter(components, plugs, bundles, directories)
1859 regwriter.dump(fd)
1860
1861 __registry = None
1862
1863
1865 """
1866 Make a bundle from a subset of all loaded modules, also writing out
1867 a registry file that can apply to that subset of the global
1868 registry. Suitable for use as a FLU_ATEXIT handler.
1869
1870 @param outfile: The path to which a zip file will be written.
1871 @type outfile: str
1872 @param outreg: The path to which a registry file will be written.
1873 @type outreg: str
1874 @param prefixes: A list of prefixes to which to limit the export. If
1875 not given, package up all modules. For example, "flumotion" would
1876 limit the output to modules that start with "flumotion".
1877 @type prefixes: list of str
1878 """
1879 from twisted.python import reflect
1880
1881 def getUsedModules(prefixes):
1882 ret = {}
1883 for modname in sys.modules:
1884 if prefixes and not filter(modname.startswith, prefixes):
1885 continue
1886 try:
1887 module = reflect.namedModule(modname)
1888 if hasattr(module, '__file__'):
1889 ret[modname] = module
1890 else:
1891 log.info('makebundle', 'Module %s has no file', module)
1892 except ImportError:
1893 log.info('makebundle', 'Could not import %s', modname)
1894 return ret
1895
1896 def calculateModuleBundleMap():
1897 allbundles = getRegistry().getBundles()
1898 ret = {}
1899 for bundle in allbundles:
1900 for directory in bundle.getDirectories():
1901 for bundleFile in directory.getFiles():
1902 path = os.path.join(directory.getName(),
1903 bundleFile.getLocation())
1904 parts = path.split(os.path.sep)
1905 if parts[-1].startswith('__init__.py'):
1906 parts.pop()
1907 elif parts[-1].endswith('.py'):
1908 parts[-1] = parts[-1][:-3]
1909 else:
1910
1911 continue
1912 modname = '.'.join(parts)
1913 ret[modname] = bundle
1914 return ret
1915
1916 def makeMergedBundler(modules, modulebundlemap):
1917 ret = MergedBundler()
1918 basket = getRegistry().makeBundlerBasket()
1919 for modname in modules:
1920 modfilename = modules[modname].__file__
1921 if modname in modulebundlemap:
1922 bundleName = modulebundlemap[modname].getName()
1923 for depBundleName in basket.getDependencies(bundleName):
1924 ret.addBundler(basket.getBundlerByName(depBundleName))
1925 else:
1926 if modfilename.endswith('.pyc'):
1927 modfilename = modfilename[:-1]
1928 if os.path.isdir(modfilename):
1929 with_init = os.path.join(modfilename, '__init__.py')
1930 if os.path.exists(with_init):
1931 modfilename = with_init
1932 nparts = len(modname.split('.'))
1933 if '__init__' in modfilename:
1934 nparts += 1
1935 relpath = os.path.join(*modfilename.split(
1936 os.path.sep)[-nparts:])
1937 ret.add(modfilename, relpath)
1938 return ret
1939
1940 modules = getUsedModules(prefixes)
1941 modulebundlemap = calculateModuleBundleMap()
1942 bundler = makeMergedBundler(modules, modulebundlemap)
1943
1944 print 'Writing bundle to', outfile
1945 open(outfile, 'w').write(bundler.bundle().getZip())
1946
1947 print 'Writing registry to', outreg
1948 bundlers_used = [b.name for b in bundler.getSubBundlers()]
1949 regwriter = RegistrySubsetWriter(onlyBundles=bundlers_used)
1950 regwriter.dump(open(outreg, 'w'))
1951
1952
1973