1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 """
19 parsing of manager configuration files
20 """
21
22 import operator
23 import warnings
24
25 from flumotion.common import log, errors, common, registry
26 from flumotion.common import config as fluconfig
27 from flumotion.common.xmlwriter import cmpComponentType, XMLWriter
28 from flumotion.configure import configure
29
30 __version__ = "$Rev$"
31
32
35
36
38
39 def parseFeedId(feedId):
40 if feedId.find(':') == -1:
41 return "%s:default" % feedId
42 else:
43 return feedId
44
45 eaterConfig = conf.get('eater', {})
46 sourceConfig = conf.get('source', [])
47 if eaterConfig == {} and sourceConfig != []:
48 eaters = registry.getRegistry().getComponent(
49 conf.get('type')).getEaters()
50 eatersDict = {}
51 eatersTuple = [(None, parseFeedId(s)) for s in sourceConfig]
52 eatersDict = buildEatersDict(eatersTuple, eaters)
53 conf['eater'] = eatersDict
54
55 if sourceConfig:
56 sources = []
57 for s in sourceConfig:
58 sources.append(parseFeedId(s))
59 conf['source'] = sources
60
61
63 eaters = dict(conf.get('eater', {}))
64 concat = lambda lists: reduce(list.__add__, lists, [])
65 if not reduce(lambda x, y: y and isinstance(x, tuple),
66 concat(eaters.values()),
67 True):
68 for eater in eaters:
69 aliases = []
70 feeders = eaters[eater]
71 for i in range(len(feeders)):
72 val = feeders[i]
73 if isinstance(val, tuple):
74 feedId, alias = val
75 aliases.append(val[1])
76 else:
77 feedId = val
78 alias = eater
79 while alias in aliases:
80 log.warning('config', "Duplicate alias %s for "
81 "eater %s, uniquifying", alias, eater)
82 alias += '-bis'
83 aliases.append(alias)
84 feeders[i] = (feedId, val)
85 conf['eater'] = eaters
86
87 UPGRADERS = [upgradeEaters, upgradeAliases]
88 CURRENT_VERSION = len(UPGRADERS)
89
90
92 """Build a eaters dict suitable for forming part of a component
93 config.
94
95 @param eatersList: List of eaters. For example,
96 [('default', 'othercomp:feeder', 'foo')] says
97 that our eater 'default' will be fed by the feed
98 identified by the feedId 'othercomp:feeder', and
99 that it has the alias 'foo'. Alias is optional.
100 @type eatersList: List of (eaterName, feedId, eaterAlias?)
101 @param eaterDefs: The set of allowed and required eaters
102 @type eaterDefs: List of
103 L{flumotion.common.registry.RegistryEntryEater}
104 @returns: Dict of eaterName => [(feedId, eaterAlias)]
105 """
106
107 def parseEaterTuple(tup):
108
109 def parse(eaterName, feedId, eaterAlias=None):
110 if eaterAlias is None:
111 eaterAlias = eaterName
112 return (eaterName, feedId, eaterAlias)
113 return parse(*tup)
114
115 eaters = {}
116 for eater, feedId, alias in [parseEaterTuple(t) for t in eatersList]:
117 if eater is None:
118 if not eaterDefs:
119 raise errors.ConfigError(
120 "Feed %r cannot be connected, component has no eaters" %
121 (feedId, ))
122
123 eater = eaterDefs[0].getName()
124 if alias is None:
125 alias = eater
126 feeders = eaters.get(eater, [])
127 if feedId in feeders:
128 raise errors.ConfigError(
129 "Already have a feedId %s eating from %s" %
130 (feedId, eater))
131 while alias in [a for f, a in feeders]:
132 log.debug('config', "Duplicate alias %s for eater %s, "
133 "uniquifying", alias, eater)
134 alias += '-bis'
135
136 feeders.append((feedId, alias))
137 eaters[eater] = feeders
138 for e in eaterDefs:
139 eater = e.getName()
140 if e.getRequired() and not eater in eaters:
141 raise errors.ConfigError("Component wants to eat on %s,"
142 " but no feeders specified."
143 % (e.getName(), ))
144 if not e.getMultiple() and len(eaters.get(eater, [])) > 1:
145 raise errors.ConfigError("Component does not support multiple "
146 "sources feeding %s (%r)"
147 % (eater, eaters[eater]))
148 aliases = reduce(list.__add__,
149 [[x[1] for x in tups] for tups in eaters.values()],
150 [])
151
152
153 while aliases:
154 alias = aliases.pop()
155 if alias in aliases:
156 raise errors.ConfigError("Duplicate alias: %s" % (alias, ))
157
158 return eaters
159
160
162 """Build a virtual feeds dict suitable for forming part of a
163 component config.
164
165 @param feedPairs: List of virtual feeds, as name-feederName pairs. For
166 example, [('bar:baz', 'qux')] defines one
167 virtual feed 'bar:baz', which is provided by
168 the component's 'qux' feed.
169 @type feedPairs: List of (feedId, feedName) -- both strings.
170 @param feeders: The feeders exported by this component, from the
171 registry.
172 @type feeders: List of str.
173 """
174 ret = {}
175 for virtual, real in feedPairs:
176 if real not in feeders:
177 raise errors.ConfigError('virtual feed maps to unknown feeder: '
178 '%s -> %s' % (virtual, real))
179 try:
180 common.parseFeedId(virtual)
181 except:
182 raise errors.ConfigError('virtual feed name not a valid feedId: %s'
183 % (virtual, ))
184 ret[virtual] = real
185 return ret
186
187
188 -def dictDiff(old, new, onlyOld=None, onlyNew=None, diff=None,
189 keyBase=None):
190 """Compute the difference between two config dicts.
191
192 @returns: 3 tuple: (onlyOld, onlyNew, diff) where:
193 onlyOld is a list of (key, value), representing key-value
194 pairs that are only in old;
195 onlyNew is a list of (key, value), representing key-value
196 pairs that are only in new;
197 diff is a list of (key, oldValue, newValue), representing
198 keys with different values in old and new; and
199 key is a tuple of strings representing the recursive key
200 to get to a value. For example, ('foo', 'bar') represents
201 the value d['foo']['bar'] on a dict d.
202 """
203
204
205 if onlyOld is None:
206 onlyOld = []
207 onlyNew = []
208 diff = []
209 keyBase = ()
210
211 for k in old:
212 key = (keyBase + (k, ))
213 if k not in new:
214 onlyOld.append((key, old[k]))
215 elif old[k] != new[k]:
216 if isinstance(old[k], dict) and isinstance(new[k], dict):
217 dictDiff(old[k], new[k], onlyOld, onlyNew, diff, key)
218 else:
219 diff.append((key, old[k], new[k]))
220
221 for k in new:
222 key = (keyBase + (k, ))
223 if k not in old:
224 onlyNew.append((key, new[k]))
225
226 return onlyOld, onlyNew, diff
227
228
231
232 def ref(label, k):
233 return "%s%s: '%s'" % (label,
234 ''.join(["[%r]" % (subk, )
235 for subk in k[:-1]]),
236 k[-1])
237
238 out = []
239 for k, v in old:
240 out.append('Only in %s = %r' % (ref(oldLabel, k), v))
241 for k, v in new:
242 out.append('Only in %s = %r' % (ref(newLabel, k), v))
243 for k, oldv, newv in diff:
244 out.append('Value mismatch:')
245 out.append(' %s = %r' % (ref(oldLabel, k), oldv))
246 out.append(' %s = %r' % (ref(newLabel, k), newv))
247 return '\n'.join(out)
248
249
250 -class ConfigEntryComponent(log.Loggable):
251 "I represent a <component> entry in a planet config file"
252 nice = 0
253 logCategory = 'config'
254
255 __pychecker__ = 'maxargs=13'
256
257 - def __init__(self, name, parent, type, label, propertyList, plugList,
258 worker, eatersList, isClockMaster, project, version,
259 virtualFeeds=None):
260 self.name = name
261 self.parent = parent
262 self.type = type
263 self.label = label
264 self.worker = worker
265 self.defs = registry.getRegistry().getComponent(self.type)
266 try:
267 self.config = self._buildConfig(propertyList, plugList,
268 eatersList, isClockMaster,
269 project, version,
270 virtualFeeds)
271 except errors.ConfigError, e:
272
273 e.args = ("While parsing component %s: %s"
274 % (name, log.getExceptionMessage(e)), )
275 raise
276
277 - def _buildVersionTuple(self, version):
278 if version is None:
279 return configure.versionTuple
280 elif isinstance(version, tuple):
281 assert len(version) == 4
282 return version
283 elif isinstance(version, str):
284 try:
285 return common.versionStringToTuple(version)
286 except:
287 raise errors.ConfigError(
288 "<component> version %r not parseable" % version)
289 raise errors.ConfigError(
290 "<component> version %r not parseable" % version)
291
292 - def _buildConfig(self, propertyList, plugsList, eatersList,
293 isClockMaster, project, version, virtualFeeds):
294 """
295 Build a component configuration dictionary.
296 """
297
298
299
300 config = {'name': self.name,
301 'parent': self.parent,
302 'type': self.type,
303 'config-version': CURRENT_VERSION,
304 'avatarId': common.componentId(self.parent, self.name),
305 'project': project or configure.PACKAGE,
306 'version': self._buildVersionTuple(version),
307 'clock-master': isClockMaster or None,
308 'feed': self.defs.getFeeders(),
309 'properties': fluconfig.buildPropertyDict(propertyList,
310 self.defs.getProperties()),
311 'plugs': fluconfig.buildPlugsSet(plugsList,
312 self.defs.getSockets()),
313 'eater': buildEatersDict(eatersList,
314 self.defs.getEaters()),
315 'source': [tup[1] for tup in eatersList],
316 'virtual-feeds': buildVirtualFeeds(virtualFeeds or [],
317 self.defs.getFeeders())}
318
319 if self.label:
320
321 config['label'] = self.label
322
323 if not config['source']:
324
325 del config['source']
326
327
328 return config
329
332
333 - def getLabel(self):
335
338
339 - def getParent(self):
341
342 - def getConfigDict(self):
344
345 - def getWorker(self):
347
348
350 """
351 I represent a <flow> entry in a planet config file.
352
353 @ivar name: name of flow
354 @type name: str
355 @ivar components: dict of name -> component config
356 @type components: dict of str -> L{ConfigEntryComponent}
357 """
358
359 - def __init__(self, name, components):
360 self.name = name
361 self.components = {}
362 for c in components:
363 if c.name in self.components:
364 raise errors.ConfigError(
365 'flow %s already has component named %s' % (name, c.name))
366 self.components[c.name] = c
367
368
370 "I represent a <manager> entry in a planet config file"
371
372 - def __init__(self, name, host, port, transport, certificate, bouncer,
373 fludebug, plugs):
374 self.name = name
375 self.host = host
376 self.port = port
377 self.transport = transport
378 self.certificate = certificate
379 self.bouncer = bouncer
380 self.fludebug = fludebug
381 self.plugs = plugs
382
383
385 "I represent a <atmosphere> entry in a planet config file"
386
387 - def __init__(self):
389
391 return len(self.components)
392
393
395 """
396 This is a base class for parsing planet configuration files (both manager
397 and flow files).
398 """
399 logCategory = 'config'
400
402 if feedId.find(':') == -1:
403 return "%s:default" % feedId
404 else:
405 return feedId
406
413
414 - def parseComponent(self, node, parent, isFeedComponent,
415 needsWorker):
416 """
417 Parse a <component></component> block.
418
419 @rtype: L{ConfigEntryComponent}
420 """
421
422
423
424
425
426
427
428
429
430
431
432
433 attrs = self.parseAttributes(node, ('name', 'type'),
434 ('label', 'worker', 'project', 'version', ))
435 name, componentType, label, worker, project, version = attrs
436 if needsWorker and not worker:
437 raise errors.ConfigError(
438 'component %s does not specify the worker '
439 'that it is to run on' % (name, ))
440 elif worker and not needsWorker:
441 raise errors.ConfigError('component %s specifies a worker to run '
442 'on, but does not need a worker' % (name, ))
443
444 properties = []
445 plugs = []
446 eaters = []
447 clockmasters = []
448 sources = []
449 virtual_feeds = []
450
451 def parseBool(node):
452 return self.parseTextNode(node, common.strToBool)
453 parsers = {'property': (self._parseProperty, properties.append),
454 'compound-property': (self._parseCompoundProperty,
455 properties.append),
456 'plugs': (self.parsePlugs, plugs.extend)}
457
458 if isFeedComponent:
459 parsers.update({'eater': (self._parseEater, eaters.extend),
460 'clock-master': (parseBool, clockmasters.append),
461 'source': (self._parseSource, sources.append),
462 'virtual-feed': (self._parseVirtualFeed,
463 virtual_feeds.append)})
464
465 self.parseFromTable(node, parsers)
466
467 if len(clockmasters) == 0:
468 isClockMaster = None
469 elif len(clockmasters) == 1:
470 isClockMaster = clockmasters[0]
471 else:
472 raise errors.ConfigError("Only one <clock-master> node allowed")
473
474 if sources:
475 msg = ('"source" tag has been deprecated in favor of "eater",'
476 ' please update your configuration file (found in'
477 ' component %r)' % name)
478 warnings.warn(msg, DeprecationWarning)
479
480 for feedId in sources:
481
482 eaters.append((None, feedId))
483
484 return ConfigEntryComponent(name, parent, componentType, label,
485 properties, plugs, worker, eaters,
486 isClockMaster, project, version,
487 virtual_feeds)
488
491
496
498
499
500
501 name, = self.parseAttributes(node, ('name', ))
502 feeds = []
503 parsers = {'feed': (self._parseFeed, feeds.append)}
504 self.parseFromTable(node, parsers)
505 if len(feeds) == 0:
506
507 raise errors.ConfigError(
508 "Eater node %s with no <feed> nodes, is not allowed" % (
509 name, ))
510 return [(name, feedId, alias) for feedId, alias in feeds]
511
512
514 """
515 I represent a planet configuration file for Flumotion.
516
517 @ivar atmosphere: A L{ConfigEntryAtmosphere}, filled in when parse() is
518 called.
519 @ivar flows: A list of L{ConfigEntryFlow}, filled in when parse() is
520 called.
521 """
522 logCategory = 'config'
523
529
549
551
552
553
554
555 ret = {}
556
557 def parseComponent(node):
558 return self.parseComponent(node, 'atmosphere', False, True)
559
560 def gotComponent(comp):
561 ret[comp.name] = comp
562 parsers = {'component': (parseComponent, gotComponent)}
563 self.parseFromTable(node, parsers)
564 return ret
565
582 parsers = {'component': (parseComponent, components.append)}
583 self.parseFromTable(node, parsers)
584
585
586
587
588 masters = [x for x in components if x.config['clock-master']]
589 if len(masters) > 1:
590 raise errors.ConfigError("Multiple clock masters in flow %s: %s"
591 % (name, ', '.join([m.name for m in masters])))
592
593 need_sync = [(x.defs.getClockPriority(), x) for x in components
594 if x.defs.getNeedsSynchronization()]
595 need_sync.sort()
596 need_sync = [x[1] for x in need_sync]
597
598 if need_sync:
599 if masters:
600 master = masters[0]
601 else:
602 master = need_sync[-1]
603
604 masterAvatarId = master.config['avatarId']
605 self.info("Setting %s as clock master" % masterAvatarId)
606
607 for c in need_sync:
608 c.config['clock-master'] = masterAvatarId
609 elif masters:
610 self.info('master clock specified, but no synchronization '
611 'necessary -- ignoring')
612 masters[0].config['clock-master'] = None
613
614 return ConfigEntryFlow(name, components)
615
616
617
619 """
620 Get all component entries from both atmosphere and all flows
621 from the configuration.
622
623 @rtype: dictionary of /parent/name -> L{ConfigEntryComponent}
624 """
625 entries = {}
626 if self.atmosphere and self.atmosphere.components:
627 for c in self.atmosphere.components.values():
628 path = common.componentId('atmosphere', c.name)
629 entries[path] = c
630
631 for flowEntry in self.flows:
632 for c in flowEntry.components.values():
633 path = common.componentId(c.parent, c.name)
634 entries[path] = c
635
636 return entries
637
638
639
640
641
642
644 """
645 I parse manager configuration out of a planet configuration file.
646
647 @ivar manager: A L{ConfigEntryManager} containing options for
648 the manager section, filled in at construction time.
649 """
650 logCategory = 'config'
651
652 MANAGER_SOCKETS = \
653 ['flumotion.component.plugs.adminaction.AdminActionPlug',
654 'flumotion.component.plugs.base.ManagerPlug',
655 'flumotion.component.plugs.identity.IdentityProviderPlug']
656
671
683
685
686
687 name, = self.parseAttributes(node, (), ('name', ))
688 ret = ConfigEntryManager(name, None, None, None, None, None,
689 None, self.plugs)
690
691 def simpleparse(proc):
692 return lambda node: self.parseTextNode(node, proc)
693
694 def recordval(k):
695
696 def record(v):
697 if getattr(ret, k):
698 raise errors.ConfigError('duplicate %s: %s'
699 % (k, getattr(ret, k)))
700 setattr(ret, k, v)
701 return record
702
703 def enum(*allowed):
704
705 def eparse(v):
706 v = str(v)
707 if v not in allowed:
708 raise errors.ConfigError('unknown value %s (should be '
709 'one of %r)' % (v, allowed))
710 return v
711 return eparse
712
713 parsers = {'host': (simpleparse(str), recordval('host')),
714 'port': (simpleparse(int), recordval('port')),
715 'transport': (simpleparse(enum('tcp', 'ssl')),
716 recordval('transport')),
717 'certificate': (simpleparse(str), recordval('certificate')),
718 'component': (_ignore, _ignore),
719 'plugs': (_ignore, _ignore),
720 'debug': (simpleparse(str), recordval('fludebug'))}
721 self.parseFromTable(node, parsers)
722 return ret
723
725
726 def parsecomponent(node):
727 return self.parseComponent(node, 'manager', False, False)
728
729 def gotcomponent(val):
730 if self.bouncer is not None:
731 raise errors.ConfigError('can only have one bouncer '
732 '(%s is superfluous)' % (val.name, ))
733
734 self.bouncer = val
735
736 def parseplugs(node):
737 return fluconfig.buildPlugsSet(self.parsePlugs(node),
738 self.MANAGER_SOCKETS)
739
740 def gotplugs(newplugs):
741 for socket in self.plugs:
742 self.plugs[socket].extend(newplugs[socket])
743
744 parsers = {'host': (_ignore, _ignore),
745 'port': (_ignore, _ignore),
746 'transport': (_ignore, _ignore),
747 'certificate': (_ignore, _ignore),
748 'component': (parsecomponent, gotcomponent),
749 'plugs': (parseplugs, gotplugs),
750 'debug': (_ignore, _ignore)}
751 self.parseFromTable(node, parsers)
752
768
770 self.doc.unlink()
771 self.doc = None
772
773
775
779
790
796
807
809 config = component.get('config')
810 attrs = [('name', component.get('name')),
811 ('type', component.get('type')),
812 ('label', config.get('label', component.get('name'))),
813 ('worker', component.get('workerRequested')),
814 ('project', config['project']),
815 ('version', common.versionTupleToString(config['version']))]
816 self.pushTag('component', attrs)
817 for name, feeders in config['eater'].items():
818 self._writeEater(name, feeders)
819 self._writeProperties(config['properties'].items())
820 if isFeedComponent:
821 if config['clock-master'] == config['avatarId']:
822 value = 'true'
823 else:
824 value = 'false'
825 self.writeTag('clock-master', data=value)
826 self._writePlugs(config['plugs'].items())
827 self._writeVirtualFeeds(config['virtual-feeds'].items())
828 self.popTag()
829 self.writeLine()
830
832 attrs = [('name', name)]
833 self.pushTag('eater', attrs)
834 for feedId, alias in feeders:
835 attrs = [('alias', alias)]
836 self.writeTag('feed', attrs, feedId)
837 self.popTag()
838
840
841 def serialise(propVal):
842 if isinstance(propVal, tuple):
843 return ["%d/%d" % propVal]
844 elif isinstance(propVal, list):
845 return propVal
846 else:
847 return [propVal]
848 for name, value in properties:
849 attrs = [('name', name)]
850 for value in serialise(value):
851 if isinstance(value, dict):
852 self.pushTag('compound-property', attrs)
853 self._writeProperties(value.items())
854 self.popTag()
855 else:
856 self.writeTag('property', attrs, value)
857
866
872
874 for name, real in virtualfeeds:
875 attrs = [('name', name),
876 ('real', real)]
877 self.writeTag('virtual-feed', attrs)
878
879
883