1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """parsing of registry, which holds component and bundle information
23 """
24
25 import os
26 import stat
27 import errno
28 import sys
29 from StringIO import StringIO
30
31 from xml.sax import saxutils
32 from twisted.spread import pb
33 from twisted.python import runtime
34
35 from flumotion.common import common, log, errors, fxml, python
36 from flumotion.common.python import makedirs
37 from flumotion.common.bundle import BundlerBasket, MergedBundler
38 from flumotion.configure import configure
39
40 __all__ = ['ComponentRegistry', 'registry']
41 __version__ = "$Rev: 8807 $"
42
43
44
45 READ_CACHE = False
46
47 FLU_RANK_NONE = 0
48
49 _VALID_WIZARD_COMPONENT_TYPES = [
50 'audio-producer',
51 'video-producer',
52 'muxer',
53 'audio-encoder',
54 'video-encoder',
55 'consumer',
56 ]
57
58 _VALID_WIZARD_PLUG_TYPES = [
59 'http-consumer',
60 'httpserver-plug',
61 ]
62
63
65 return os.stat(file)[stat.ST_MTIME]
66
67
68 -class RegistryEntryScenario(pb.Copyable, pb.RemoteCopy):
69 """
70 I represent a <scenario> entry in the registry
71 """
72
73 - def __init__(self, type, description, base, entries):
74 """
75 @param type: the type of this scenario
76 @type type: str
77 @param description: description of this scenario
78 @type description: str
79 @param base: base directory where this scenario is placed
80 @type base: str
81 @param entries: dict of entry point type -> entry
82 @type entries: dict of str -> L{RegistryEntryEntry}
83 """
84 self.type = type
85
86 self.description = description or ""
87 self.base = base
88 self.entries = entries
89
90 - def getEntries(self):
91 """
92 Get the entries asociated with this scenario
93
94 @rtype: list of L{RegistryEntryEntry}
95 """
96 return self.entries.values()
97
98 - def getEntryByType(self, type):
99 """
100 Get the entry point for the given type of entry.
101
102 @param type: The type of the wanted entry.
103 @type type: string
104
105 @rtype: L{RegistryEntryEntry}
106 """
107 return self.entries[type]
108
111
114
115 - def getDescription(self):
116 return self.description
117
118 pb.setUnjellyableForClass(RegistryEntryScenario, RegistryEntryScenario)
119
120
121 -class RegistryEntryComponent(pb.Copyable, pb.RemoteCopy):
122 """
123 I represent a <component> entry in the registry
124 """
125
126
127 __pychecker__ = 'maxargs=15'
128
129 - def __init__(self, filename, type,
130 source, description, base, properties, files,
131 entries, eaters, feeders, needs_sync, clock_priority,
132 sockets, wizards):
133 """
134 @param filename: name of the XML file this component is parsed from
135 @type filename: str
136 @param properties: dict of name -> property
137 @type properties: dict of str -> L{RegistryEntryProperty}
138 @param files: list of files
139 @type files: list of L{RegistryEntryFile}
140 @param entries: dict of entry point type -> entry
141 @type entries: dict of str -> L{RegistryEntryEntry}
142 @param sockets: list of sockets supported by the component
143 @type sockets: list of str
144 @param wizards: list of wizard entries
145 @type wizards: list of L{RegistryEntryWizard}
146 """
147 self.filename = filename
148 self.type = type
149 self.source = source
150 self.description = description
151
152 if not self.description:
153 self.description = ""
154 self.base = base
155 self.properties = properties
156 self.files = files
157 self.entries = entries
158 self.eaters = eaters
159 self.feeders = feeders
160 self.needs_sync = needs_sync
161 self.clock_priority = clock_priority
162 self.sockets = sockets
163 self.wizards = wizards
164
165 - def getProperties(self):
166 """
167 Get a list of all properties.
168
169 @rtype: list of L{RegistryEntryProperty}
170 """
171 return self.properties.values()
172
173 - def hasProperty(self, name):
174 """
175 Check if the component has a property with the given name.
176 """
177 return name in self.properties.keys()
178
179 - def getFiles(self):
180 """
181 @rtype: list of L{RegistryEntryFile}
182 """
183 return self.files
184
185 - def getEntries(self):
186 return self.entries.values()
187
188 - def getEntryByType(self, type):
189 """
190 Get the entry point for the given type of entry.
191
192 @type type: string
193 """
194 return self.entries[type]
195
196 - def getGUIEntry(self):
197 if not self.files:
198 return
199
200
201 if len(self.files) > 1:
202 return
203
204 return self.files[0].getFilename()
205
208
211
212 - def getDescription(self):
213 return self.description
214
215 - def getSource(self):
217
218 - def getEaters(self):
220
221 - def getFeeders(self):
223
225 return self.needs_sync
226
228 return self.clock_priority
229
230 - def getSockets(self):
232 pb.setUnjellyableForClass(RegistryEntryComponent, RegistryEntryComponent)
233
234
236 """
237 I represent a <plug> entry in the registry
238 """
239
240 - def __init__(self, filename, type,
241 description, socket, entries, properties, wizards):
242 """
243 @param filename: name of the XML file this plug is parsed from
244 @type filename: str
245 @param type: the type of plug
246 @type type: str
247 @param description: the translatable description of the plug
248 @type description: str
249 @param socket: the fully qualified class name of the socket this
250 plug can be plugged in to
251 @type socket: str
252 @param entries: entry points for instantiating the plug
253 @type entries: list of L{RegistryEntryEntry}
254 @param properties: properties of the plug
255 @type properties: dict of str -> L{RegistryEntryProperty}
256 @param wizards: list of wizard entries
257 @type wizards: list of L{RegistryEntryWizard}
258 """
259 self.filename = filename
260 self.type = type
261 self.description = description
262 self.socket = socket
263 self.entries = entries
264 self.properties = properties
265 self.wizards = wizards
266
267 - def getProperties(self):
268 """
269 Get a list of all properties.
270
271 @rtype: list of L{RegistryEntryProperty}
272 """
273 return self.properties.values()
274
275 - def hasProperty(self, name):
276 """
277 Check if the component has a property with the given name.
278 """
279 return name in self.properties.keys()
280
281 - def getEntryByType(self, type):
282 """
283 Get the entry point for the given type of entry.
284
285 @type type: string
286 """
287 return self.entries[type]
288
289 - def getEntry(self):
290 return self.entries['default']
291
292 - def getEntries(self):
293 return self.entries.values()
294
297
298 - def getDescription(self):
299 return self.description
300
301 - def getSocket(self):
303
304
306 "This class represents a <bundle> entry in the registry"
307
308 - def __init__(self, name, project, under, dependencies, directories):
309 self.name = name
310 self.project = project
311 self.under = under
312 self.dependencies = dependencies
313 self.directories = directories
314
315 - def __repr__(self):
316 return '<Bundle name=%s>' % self.name
317
320
321 - def getDependencies(self):
322 """
323 @rtype: list of str
324 """
325 return self.dependencies
326
327 - def getDirectories(self):
328 """
329 @rtype: list of L{RegistryEntryBundleDirectory}
330 """
331 return self.directories
332
333 - def getProject(self):
335
336 - def getUnder(self):
338
339 - def getBaseDir(self):
340 if self.project == configure.PACKAGE:
341 return getattr(configure, self.under)
342
343 from flumotion.project import project
344 return project.get(self.project, self.under)
345
346
348 "This class represents a <directory> entry in the registry"
349
350 - def __init__(self, name, files):
351 self.name = name
352 self.files = files
353
356
357 - def getFiles(self):
359
360
362 "This class represents a <filename> entry in the registry"
363
364 - def __init__(self, location, relative):
365 self.location = location
366 self.relative = relative
367
368 - def getLocation(self):
370
371 - def getRelative(self):
373
374
376 "This class represents a <property> entry in the registry"
377
378 - def __init__(self, name, type, description,
379 required=False, multiple=False):
380 self.name = name
381 self.type = type
382 self.description = description
383
384 if not self.description:
385 self.description = ""
386 self.required = required
387 self.multiple = multiple
388
389 - def __repr__(self):
390 return '<Property name=%s>' % self.name
391
394
397
398 - def getDescription(self):
399 return self.description
400
401 - def isRequired(self):
403
404 - def isMultiple(self):
406
407
408 -class RegistryEntryCompoundProperty(RegistryEntryProperty):
409 "This class represents a <compound-property> entry in the registry"
410
411 - def __init__(self, name, description, properties, required=False,
412 multiple=False):
416
417 - def __repr__(self):
418 return '<Compound-property name=%s>' % self.name
419
420 - def getProperties(self):
421 """
422 Get a list of all sub-properties.
423
424 @rtype: list of L{RegistryEntryProperty}
425 """
426 return self.properties.values()
427
428 - def hasProperty(self, name):
429 """
430 Check if the compound-property has a sub-property with the
431 given name.
432 """
433 return name in self.properties
434
435
437 "This class represents a <file> entry in the registry"
438
439 - def __init__(self, filename, type):
440 self.filename = filename
441 self.type = type
442
444 return os.path.basename(self.filename)
445
448
449 - def getFilename(self):
451
452 - def isType(self, type):
453 return self.type == type
454
455
457 "This class represents a <entry> entry in the registry"
458
459 - def __init__(self, type, location, function):
460 self.type = type
461 self.location = location
462 self.function = function
463
466
467 - def getLocation(self):
469
470 - def getModuleName(self, base=None):
471 if base:
472 path = os.path.join(base, self.getLocation())
473 else:
474 path = self.getLocation()
475 return common.pathToModuleName(path)
476
477 - def getFunction(self):
479
480
482 "This class represents a <eater> entry in the registry"
483
484 - def __init__(self, name, required=True, multiple=False):
485 self.name = name
486 self.required = required
487 self.multiple = multiple
488
491
492 - def getRequired(self):
494
495 - def getMultiple(self):
497
498
499 -class RegistryEntryWizard(pb.Copyable):
500 "This class represents a <wizard> entry in the registry"
501
502 - def __init__(self, componentType, type, description, feeder,
503 eater, accepts, provides, rank=FLU_RANK_NONE):
504 self.componentType = componentType
505 self.type = type
506 self.description = description
507 self.feeder = feeder
508 self.eater = eater
509 self.accepts = accepts
510 self.provides = provides
511 self.rank = rank
512
513 - def __repr__(self):
514 return '<wizard %s type=%s, feeder=%s>' % (self.componentType,
515 self.type, self.feeder)
516
517
519 """
520 This class represents an <accept-format> or <provide-format>
521 entry in the registry
522 """
523
525 self.media_type = media_type
526
527
529 """
530 Registry parser
531
532 I have two modes, one to parse registries and another one to parse
533 standalone component files.
534
535 For parsing registries use the parseRegistry function and for components
536 use parseRegistryFile.
537
538 I also have a list of all components and directories which the
539 registry uses (instead of saving its own copy)
540 """
541
544
546 self._components = {}
547 self._directories = {}
548 self._bundles = {}
549 self._plugs = {}
550 self._scenarios = {}
551
553 return self._components.values()
554
561
563 return self._scenarios.values()
564
566 if type in self._scenarios:
567 return self._scenarios[type]
568 return None
569
571 return self._plugs.values()
572
579
581
582
583
584
585 components = {}
586
587 def addComponent(comp):
588 components[comp.getType()] = comp
589
590 parsers = {'component': (self._parseComponent, addComponent)}
591 self.parseFromTable(node, parsers)
592
593 return components
594
596
597
598
599
600
601
602
603
604
605
606
607
608 componentType, baseDir, description, _description = \
609 self.parseAttributes(node,
610 required=('type', 'base'),
611 optional=('description', '_description'))
612
613
614 if description:
615 import warnings
616 warnings.warn(
617 "Please change '<component description=...'"
618 " to '<component _description=...' for %s" % componentType,
619 DeprecationWarning)
620 if _description:
621 description = _description
622
623 files = []
624 source = fxml.Box(None)
625 entries = {}
626 eaters = []
627 feeders = []
628 synchronization = fxml.Box((False, 100))
629 sockets = []
630 properties = {}
631 wizards = []
632
633
634
635
636
637
638
639
640 parsers = {
641 'source': (self._parseSource, source.set),
642 'properties': (self._parseProperties, properties.update),
643 'files': (self._parseFiles, files.extend),
644 'entries': (self._parseEntries, entries.update),
645 'eater': (self._parseEater, eaters.append),
646 'feeder': (self._parseFeeder, feeders.append),
647 'synchronization': (self._parseSynchronization,
648 synchronization.set),
649 'sockets': (self._parseSockets, sockets.extend),
650 'wizard': (self._parseComponentWizard, wizards.append),
651 }
652 self.parseFromTable(node, parsers)
653
654 source = source.unbox()
655 needs_sync, clock_priority = synchronization.unbox()
656
657 return RegistryEntryComponent(self.filename,
658 componentType, source, description,
659 baseDir, properties, files,
660 entries, eaters, feeders,
661 needs_sync, clock_priority,
662 sockets, wizards)
663
665
666
667
668
669 scenarios = {}
670
671 def addScenario(scenario):
672 scenarios[scenario.getType()] = scenario
673
674 parsers = {'scenario': (self._parseScenario, addScenario)}
675 self.parseFromTable(node, parsers)
676
677 return scenarios
678
680
681
682
683
684 scenarioType, baseDir, description = \
685 self.parseAttributes(node,
686 required=('type', 'base'),
687 optional=('_description', ))
688
689 entries = {}
690
691 parsers = {
692 'entries': (self._parseEntries, entries.update),
693 }
694
695 self.parseFromTable(node, parsers)
696
697 return RegistryEntryScenario(scenarioType, description,
698 baseDir, entries)
699
704
706
707
708
709
710 attrs = self.parseAttributes(node, required=('name', 'type'),
711 optional=('required', 'multiple', 'description', '_description'))
712 name, propertyType, required, multiple, description, _d = attrs
713 if description:
714 import warnings
715 warnings.warn("Please change '<property description=...'"
716 " to '<property _description=...' for %s" % name,
717 DeprecationWarning)
718 if _d:
719 description = _d
720
721
722 allowed = ('string', 'rawstring', 'int', 'long', 'bool',
723 'float', 'fraction')
724 if propertyType not in allowed:
725 raise fxml.ParserError(
726 "<property> %s's type is not one of %s" % (
727 name, ", ".join(allowed)))
728 required = common.strToBool(required)
729 multiple = common.strToBool(multiple)
730 return RegistryEntryProperty(name, propertyType, description,
731 required=required, multiple=multiple)
732
734
735
736
737
738
739
740
741 attrs = self.parseAttributes(node, required=('name', ),
742 optional=('required', 'multiple', 'description', '_description'))
743 name, required, multiple, description, _description = attrs
744 if description:
745 import warnings
746 warnings.warn("Please change '<compound-property description=...'"
747 " to '<compound-property _description=...' for %s" % name,
748 DeprecationWarning)
749 if _description:
750 description = _description
751
752
753 required = common.strToBool(required)
754 multiple = common.strToBool(multiple)
755
756 properties = {}
757
758 def addProperty(prop):
759 properties[prop.getName()] = prop
760
761 parsers = {'property': (self._parseProperty, addProperty),
762 'compound-property': (self._parseCompoundProperty,
763 addProperty)}
764 self.parseFromTable(node, parsers)
765
766 return RegistryEntryCompoundProperty(name, description, properties,
767 required=required, multiple=multiple)
768
779
780 parsers = {'property': (self._parseProperty, addProperty),
781 'compound-property': (self._parseCompoundProperty,
782 addProperty)}
783
784 self.parseFromTable(node, parsers)
785
786 return properties
787
796
808
815
827
828 - def _parseEntry(self, node):
829 attrs = self.parseAttributes(node, ('type', 'location', 'function'))
830 entryType, location, function = attrs
831 return RegistryEntryEntry(entryType, location, function)
832
834
835
836
837
838
839 entries = {}
840
841 def addEntry(entry):
842 if entry.getType() in entries:
843 raise fxml.ParserError("entry %s already specified"
844 % entry.getType())
845 entries[entry.getType()] = entry
846
847 parsers = {'entry': (self._parseEntry, addEntry)}
848
849 self.parseFromTable(node, parsers)
850
851 return entries
852
863
868
876
877 - def _parsePlugEntry(self, node):
878 attrs = self.parseAttributes(node,
879 ('location', 'function'), ('type', ))
880 location, function, entryType = attrs
881 if not entryType:
882 entryType = 'default'
883 return RegistryEntryEntry(entryType, location, function)
884
886 return {'default': self._parsePlugEntry(node)}
887
889
890
891
892
893
894 entries = {}
895
896 def addEntry(entry):
897 if entry.getType() in entries:
898 raise fxml.ParserError("entry %s already specified"
899 % entry.getType())
900 entries[entry.getType()] = entry
901
902 parsers = {'entry': (self._parsePlugEntry, addEntry)}
903
904 self.parseFromTable(node, parsers)
905
906 return entries
907
909
910
911
912
913
914
915
916
917 plugType, socket, description = \
918 self.parseAttributes(node, required=('type', 'socket'),
919 optional=('_description', ))
920
921 if not description:
922 import warnings
923 warnings.warn(
924 "Please add '_description=...' attribute to plug '%s'" %
925 plugType,
926 DeprecationWarning)
927 description = 'TODO'
928
929 entries = {}
930 properties = {}
931 wizards = []
932
933 parsers = {
934 'entries': (self._parsePlugEntries, entries.update),
935
936 'entry': (self._parseDefaultPlugEntry, entries.update),
937 'properties': (self._parseProperties, properties.update),
938 'wizard': (self._parsePlugWizard, wizards.append),
939 }
940
941 self.parseFromTable(node, parsers)
942
943 if not 'default' in entries:
944 raise fxml.ParserError(
945 "<plug> %s needs a default <entry>" % plugType)
946
947 return RegistryEntryPlug(self.filename, plugType, description,
948 socket, entries, properties,
949 wizards)
950
962
963 parsers = {'plug': (self._parsePlug, addPlug)}
964 self.parseFromTable(node, parsers)
965
966 return plugs
967
968
969
971 """
972 @param file: The file to parse, either as an open file object,
973 or as the name of a file to open.
974 @type file: str or file.
975 """
976 if isinstance(file, basestring):
977 self.filename = file
978 else:
979 self.filename = getattr(file, 'name', '<string>')
980 root = self.getRoot(file)
981 node = root.documentElement
982
983 if node.nodeName != 'registry':
984
985
986 self.debug('%s does not have registry as root tag', self.filename)
987 return
988
989
990 self._parseRoot(node, disallowed=['directories'])
991 root.unlink()
992
1002
1003 parsers = {'bundle': (self._parseBundle, addBundle)}
1004 self.parseFromTable(node, parsers)
1005
1006 return bundles
1007
1009
1010
1011
1012
1013
1014 attrs = self.parseAttributes(node, ('name', ), ('project', 'under'))
1015 name, project, under = attrs
1016 project = project or configure.PACKAGE
1017 under = under or 'pythondir'
1018
1019 dependencies = []
1020 directories = []
1021
1022 parsers = {'dependencies': (self._parseBundleDependencies,
1023 dependencies.extend),
1024 'directories': (self._parseBundleDirectories,
1025 directories.extend)}
1026 self.parseFromTable(node, parsers)
1027
1028 return RegistryEntryBundle(name, project, under,
1029 dependencies, directories)
1030
1034
1046
1058
1067
1078
1079 parsers = {'filename': (parseFilename, filenames.append)}
1080 self.parseFromTable(node, parsers)
1081
1082 return RegistryEntryBundleDirectory(name, filenames)
1083
1084
1085
1087 """
1088 @param file: The file to parse, either as an open file object,
1089 or as the name of a file to open.
1090 @type file: str or file.
1091 """
1092 if isinstance(file, basestring):
1093 self.filename = file
1094 else:
1095 self.filename = getattr(file, 'name', '<string>')
1096 root = self.getRoot(file)
1097 self._parseRoot(root.documentElement)
1098 root.unlink()
1099
1101 return self._directories.values()
1102
1104 return self._directories[name]
1105
1107 """
1108 Add a registry path object to the parser.
1109
1110 @type directory: {RegistryDirectory}
1111 """
1112 self._directories[directory.getPath()] = directory
1113
1115 """
1116 Remove a directory from the parser given the path.
1117 Used when the path does not actually contain any registry information.
1118 """
1119 if path in self._directories.keys():
1120 del self._directories[path]
1121
1123
1124
1125
1126
1127
1128 parsers = {'components': (self._parseComponents,
1129 self._components.update),
1130 'directories': (self._parseDirectories,
1131 self._directories.update),
1132 'bundles': (self._parseBundles, self._bundles.update),
1133 'plugs': (self._parsePlugs, self._plugs.update),
1134 'scenarios': (self._parseScenarios, self._scenarios.update)}
1135
1136 if disallowed:
1137 for k in disallowed:
1138 del parsers[k]
1139
1140 self.parseFromTable(node, parsers)
1141
1143
1144
1145
1146
1147 directories = {}
1148
1149 def addDirectory(d):
1150 directories[d.getPath()] = d
1151
1152 parsers = {'directory': (self._parseDirectory, addDirectory)}
1153 self.parseFromTable(node, parsers)
1154
1155 return directories
1156
1161
1164
1167
1169
1170
1171
1172
1173
1174
1175 attrs = self.parseAttributes(node,
1176 ('type', '_description'),
1177 ('feeder', 'eater', 'rank'))
1178 wizardType, description, feeder, eater, rank = attrs
1179
1180 accepts = []
1181 provides = []
1182 parsers = {
1183 'accept-format': (self._parseAcceptFormat,
1184 lambda n: accepts.append(n)),
1185 'provide-format': (self._parseProvideFormat,
1186 lambda n: provides.append(n)),
1187 }
1188 self.parseFromTable(node, parsers)
1189
1190 parent_type = node.parentNode.getAttribute('type')
1191
1192 if not wizardType in validTypes:
1193 raise fxml.ParserError(
1194 "<wizard>'s type attribute is %s must be one of %s" % (
1195 parent_type,
1196 ', '.join(validTypes)))
1197 rank = int(rank or FLU_RANK_NONE)
1198 isProducer = wizardType.endswith('-producer')
1199 isEncoder = wizardType.endswith('-encoder')
1200 isMuxer = (wizardType == 'muxer')
1201 isConsumer = wizardType.endswith('-consumer')
1202
1203 err = None
1204
1205 if accepts and (isProducer or isEncoder):
1206 err = ('<wizard type="%s"> does not allow an accepted '
1207 'media-type.') % (parent_type, )
1208
1209 elif not accepts and (isMuxer or isConsumer):
1210 err = ('<wizard type="%s"> requires at least one accepted '
1211 'media-type.') % (parent_type, )
1212
1213 elif provides and (isProducer or isConsumer):
1214 err = ('<wizard type="%s"> does not allow a provided '
1215 'media-type.') % (parent_type, )
1216
1217 if len(provides) != 1 and (isEncoder or isMuxer):
1218 err = ('<wizard type="%s"> requires exactly one provided '
1219 'media-type.') % (parent_type, )
1220
1221 if err:
1222 raise fxml.ParserError(err)
1223
1224 return RegistryEntryWizard(parent_type, wizardType, description,
1225 feeder, eater, accepts, provides, rank)
1226
1231
1236
1237
1238
1239
1240
1242 """
1243 I represent a directory under a path managed by the registry.
1244 I can be queried for a list of partial registry .xml files underneath
1245 the given path, under the given prefix.
1246 """
1247
1253
1255 return "<RegistryDirectory %s>" % self._path
1256
1258 """
1259 Get all files ending in .xml from all directories under the given root.
1260
1261 @type root: string
1262 @param root: the root directory under which to search
1263
1264 @returns: a list of .xml files, relative to the given root directory
1265 """
1266 files = []
1267 dirs = []
1268
1269 if os.path.exists(root):
1270 try:
1271 directory_files = os.listdir(root)
1272 except OSError, e:
1273 if e.errno == errno.EACCES:
1274 return files, dirs
1275 else:
1276 raise
1277
1278 dirs.append(root)
1279
1280 for entry in directory_files:
1281 path = os.path.join(root, entry)
1282
1283 if not os.path.isdir(path):
1284 if path.endswith('.xml'):
1285 files.append(path)
1286
1287
1288 elif entry != '.svn':
1289 newFiles, newDirs = self._getFileLists(path)
1290 files.extend(newFiles)
1291 dirs.extend(newDirs)
1292
1293 return files, dirs
1294
1296
1297 def _rebuildNeeded(file):
1298 try:
1299 if _getMTime(file) > mtime:
1300 self.debug("Path %s changed since registry last "
1301 "scanned", f)
1302 return True
1303 return False
1304 except OSError:
1305 self.debug("Failed to stat file %s, need to rescan", f)
1306 return True
1307
1308 for f in self._files:
1309 if _rebuildNeeded(f):
1310 return True
1311 for f in self._dirs:
1312 if _rebuildNeeded(f):
1313 return True
1314 return False
1315
1317 """
1318 Return a list of all .xml registry files underneath this registry
1319 path.
1320 """
1321 return self._files
1322
1325
1326
1328
1329 - def __init__(self, components, plugs, bundles, directories):
1330 """
1331 @param components: components to write
1332 @type components: list of L{RegistryEntryComponent}
1333 @param plugs: plugs to write
1334 @type plugs: list of L{RegistryEntryPlug}
1335 @param bundles: bundles to write
1336 @type bundles: list of L{RegistryEntryBundle}
1337 @param directories: directories to write
1338 @type directories: list of L{RegistryEntryBundleDirectory}
1339 """
1340 self.components = components
1341 self.plugs = plugs
1342 self.bundles = bundles
1343 self.directories = directories
1344
1345 - def dump(self, fd):
1346 """
1347 Dump the cache of components to the given opened file descriptor.
1348
1349 @type fd: integer
1350 @param fd: open file descriptor to write to
1351 """
1352
1353 def w(i, msg):
1354 print >> fd, ' '*i + msg
1355
1356 def e(attr):
1357 return saxutils.quoteattr(attr)
1358
1359 def _dump_proplist(i, proplist, ioff=2):
1360 for prop in proplist:
1361 if isinstance(prop, RegistryEntryCompoundProperty):
1362 _dump_compound(i, prop)
1363 else:
1364 w(i, ('<property name="%s" type="%s"'
1365 % (prop.getName(), prop.getType())))
1366 w(i, (' _description=%s'
1367 % (e(prop.getDescription()), )))
1368 w(i, (' required="%s" multiple="%s"/>'
1369 % (prop.isRequired(), prop.isMultiple())))
1370
1371 def _dump_compound(i, cprop, ioff=2):
1372 w(i, ('<compound-property name="%s"' % (cprop.getName(), )))
1373 w(i, (' _description=%s'
1374 % (e(cprop.getDescription()), )))
1375 w(i, (' required="%s" multiple="%s">'
1376 % (cprop.isRequired(), cprop.isMultiple())))
1377 _dump_proplist(i + ioff, cprop.getProperties())
1378 w(i, ('</compound-property>'))
1379
1380 def _dump_entries(i, entries):
1381 if not entries:
1382 return
1383
1384 w(i, '<entries>')
1385 for entry in entries:
1386 w(i+2, '<entry type="%s" location="%s" function="%s"/>' % (
1387 entry.getType(),
1388 entry.getLocation(),
1389 entry.getFunction()))
1390 w(i, '</entries>')
1391
1392 w(0, '<registry>')
1393 w(0, '')
1394
1395
1396 w(2, '<components>')
1397 w(0, '')
1398 for component in self.components:
1399 w(4, '<component type="%s" base="%s"' % (
1400 component.getType(), component.getBase()))
1401 w(4, ' _description=%s>'
1402 % (e(component.getDescription()), ))
1403
1404 w(6, '<source location="%s"/>' % component.getSource())
1405 for x in component.getEaters():
1406 w(6, '<eater name="%s" required="%s" multiple="%s"/>'
1407 % (x.getName(), x.getRequired() and "yes" or "no",
1408 x.getMultiple() and "yes" or "no"))
1409 for x in component.getFeeders():
1410 w(6, '<feeder name="%s"/>' % x)
1411 w(6, '<synchronization required="%s" clock-priority="%d"/>'
1412 % (component.getNeedsSynchronization() and "yes" or "no",
1413 component.getClockPriority()))
1414
1415 sockets = component.getSockets()
1416 if sockets:
1417 w(6, '<sockets>')
1418 for socket in sockets:
1419 w(8, '<socket type="%s"/>' % socket)
1420 w(6, '</sockets>')
1421
1422 w(6, '<properties>')
1423 _dump_proplist(8, component.getProperties())
1424 w(6, '</properties>')
1425
1426 for wizard in component.wizards:
1427 rank = ''
1428 if wizard.rank:
1429 rank = ' rank="%d"' % wizard.rank
1430 w(6, '<wizard type="%s" _description="%s" feeder="%s"%s>' % (
1431 wizard.type,
1432 e(wizard.description),
1433 wizard.feeder,
1434 rank))
1435 for accept in wizard.accepts:
1436 w(8, '<accept-format media-type="%s"/>' % (
1437 accept.media_type))
1438 for provide in wizard.provides:
1439 w(8, '<provide-format media-type="%s"/>' % (
1440 provide.media_type))
1441 w(6, '</wizard>')
1442
1443 registryEntryFiles = component.getFiles()
1444 if registryEntryFiles:
1445 w(6, '<files>')
1446 for entryFile in registryEntryFiles:
1447 w(8, '<file name="%s" type="%s"/>' % (
1448 entryFile.getName(),
1449 entryFile.getType()))
1450 w(6, '</files>')
1451
1452 _dump_entries(6, component.getEntries())
1453
1454 w(4, '</component>')
1455 w(0, '')
1456
1457 w(2, '</components>')
1458 w(0, '')
1459
1460
1461 w(2, '<plugs>')
1462 w(0, '')
1463 for plug in self.plugs:
1464 w(4, '<plug type="%s" socket="%s" _description="%s">'
1465 % (plug.getType(), plug.getSocket(), plug.getDescription()))
1466
1467 _dump_entries(6, plug.getEntries())
1468
1469 w(6, '<properties>')
1470 _dump_proplist(8, plug.getProperties())
1471 w(6, '</properties>')
1472
1473 w(4, '</plug>')
1474 w(0, '')
1475
1476 w(2, '</plugs>')
1477 w(0, '')
1478
1479
1480 w(2, '<bundles>')
1481 for bundle in self.bundles:
1482 w(4, '<bundle name="%s" under="%s" project="%s">' % (
1483 bundle.getName(), bundle.getUnder(), bundle.getProject()))
1484
1485 dependencies = bundle.getDependencies()
1486 if dependencies:
1487 w(6, '<dependencies>')
1488 for dependency in dependencies:
1489 w(8, '<dependency name="%s"/>' % dependency)
1490 w(6, '</dependencies>')
1491
1492 bundleDirectories = bundle.getDirectories()
1493 if bundleDirectories:
1494 w(6, '<directories>')
1495 for directory in bundleDirectories:
1496 w(8, '<directory name="%s">' % directory.getName())
1497 for filename in directory.getFiles():
1498 w(10, '<filename location="%s" relative="%s"/>' % (
1499 filename.getLocation(), filename.getRelative()))
1500 w(8, '</directory>')
1501 w(6, '</directories>')
1502
1503 w(4, '</bundle>')
1504 w(0, '')
1505 w(2, '</bundles>')
1506
1507
1508
1509 directories = self.directories
1510 if directories:
1511 w(2, '<directories>')
1512 w(0, '')
1513 for d in directories:
1514 w(4, '<directory filename="%s"/>' % d.getPath())
1515 w(2, '</directories>')
1516 w(0, '')
1517
1518 w(0, '</registry>')
1519
1520
1522 """Registry, this is normally not instantiated."""
1523
1524 logCategory = 'registry'
1525 defaultCachePath = os.path.join(configure.registrydir, 'registry.xml')
1526
1556
1558 """
1559 @param file: The file to add, either as an open file object, or
1560 as the name of a file to open.
1561 @type file: str or file.
1562 """
1563 if isinstance(file, str) and file.endswith('registry.xml'):
1564 self.warning('%s seems to be an old registry in your tree, '
1565 'please remove it', file)
1566 self.debug('Adding file: %r', file)
1567 self._parser.parseRegistryFile(file)
1568
1570 f = StringIO(string)
1571 self.addFile(f)
1572 f.close()
1573
1575 """
1576 Add a registry path to this registry, scanning it for registry
1577 snippets.
1578
1579 @param path: a full path containing a PREFIX directory, which will be
1580 scanned for registry files.
1581 @param prefix: directory name under path which will be scanned
1582 (defaults to 'flumotion' and cannot be an empty string).
1583
1584 @rtype: bool
1585 @returns: whether the path could be added
1586 """
1587 prefix = prefix or self.prefix
1588 self.debug('path %s, prefix %s', path, prefix)
1589 if not os.path.exists(path):
1590 self.warning(
1591 "Cannot add non-existent path '%s' to registry", path)
1592 return False
1593 if not os.path.exists(os.path.join(path, prefix)):
1594 self.warning("Cannot add path '%s' to registry "
1595 "since it does not contain prefix '%s'", path, prefix)
1596 return False
1597
1598
1599
1600 self.info('Scanning registry path %s', path)
1601 registryPath = RegistryDirectory(path, prefix=prefix)
1602 files = registryPath.getFiles()
1603 self.debug('Found %d possible registry files', len(files))
1604 map(self.addFile, files)
1605
1606 self._parser.addDirectory(registryPath)
1607 return True
1608
1609
1610
1611
1613 return len(self._parser._components) == 0
1614
1616 """
1617 @rtype: L{RegistryEntryComponent}
1618 """
1619 return self._parser.getComponent(name)
1620
1622 return name in self._parser._components
1623
1626
1628 """
1629 @rtype: L{RegistryEntryPlug}
1630 """
1631 return self._parser.getPlug(type)
1632
1634 return name in self._parser._plugs
1635
1638
1641
1644
1646 return self._parser._bundles.values()
1647
1650
1652 """
1653 @rtype: L{flumotion.common.bundle.BundlerBasket}
1654 """
1655
1656 def load():
1657 ret = BundlerBasket()
1658 for b in self.getBundles():
1659 bundleName = b.getName()
1660 self.debug('Adding bundle %s', bundleName)
1661 for d in b.getDirectories():
1662 directory = d.getName()
1663 for bundleFilename in d.getFiles():
1664 try:
1665 basedir = b.getBaseDir()
1666 except errors.NoProjectError, e:
1667 self.warning("Could not load project %s", e.args)
1668 raise
1669 fullpath = os.path.join(basedir, directory,
1670 bundleFilename.getLocation())
1671 relative = bundleFilename.getRelative()
1672 self.log('Adding path %s as %s to bundle %s',
1673 fullpath, relative, bundleName)
1674 try:
1675 ret.add(bundleName, fullpath, relative)
1676 except Exception, e:
1677 self.debug("Reason: %r", e)
1678 raise RuntimeError(
1679 'Could not add %s to bundle %s (%s)'
1680 % (fullpath, bundleName, e))
1681 for d in b.getDependencies():
1682 self.log('Adding dependency of %s on %s', bundleName, d)
1683 ret.depend(bundleName, d)
1684 return ret
1685
1686 try:
1687 return load()
1688 except Exception, e:
1689 self.debug("Could not register bundles the first time: %s",
1690 log.getExceptionMessage(e))
1691 self.warning("Bundle problem, rebuilding registry")
1692 self.verify(force=True)
1693 try:
1694 return load()
1695 except Exception, e:
1696 self.debug("Could not register bundles the second time: %s",
1697 log.getExceptionMessage(e))
1698 self.error("Could not not register bundles (%s)",
1699 log.getExceptionMessage(e))
1700
1701 - def dump(self, fd):
1711
1713 """
1714 Clean the cache of components.
1715 """
1716 self._parser.clean()
1717
1719 if self.mtime is None or not os.path.exists(self.filename):
1720 return True
1721
1722
1723
1724 registryPaths = python.set(self._paths)
1725 oldRegistryPaths = python.set([directory.getPath()
1726 for directory in self.getDirectories()])
1727 if registryPaths != oldRegistryPaths:
1728 if oldRegistryPaths - registryPaths:
1729 return True
1730 if filter(os.path.exists, registryPaths - oldRegistryPaths):
1731 return True
1732
1733 registry_modified = self.mtime
1734 for d in self._parser.getDirectories():
1735 if d.rebuildNeeded(registry_modified):
1736 return True
1737
1738 return False
1739
1740 - def save(self, force=False):
1741 if not force and not self.rebuildNeeded():
1742 return
1743
1744 self.info('Saving registry to %s', self.filename)
1745
1746
1747 directory = os.path.split(self.filename)[0]
1748 if not os.path.exists(directory):
1749 try:
1750 makedirs(directory)
1751 except OSError, e:
1752 if e.errno == errno.EACCES:
1753 self.error('Registry directory %s could not be created !' %
1754 directory)
1755 else:
1756 raise
1757
1758 if not os.path.isdir(directory):
1759 self.error('Registry directory %s is not a directory !')
1760 try:
1761 fd = open(self.filename, 'w')
1762 self.dump(fd)
1763 except IOError, e:
1764 if e.errno == errno.EACCES:
1765 self.error('Registry file %s could not be created !' %
1766 self.filename)
1767 else:
1768 raise
1769
1771 registryPaths = [configure.pythondir, ]
1772 if 'FLU_PROJECT_PATH' in os.environ:
1773 paths = os.environ['FLU_PROJECT_PATH']
1774 registryPaths += paths.split(':')
1775 return registryPaths
1776
1777 - def verify(self, force=False):
1778 """
1779 Verify if the registry is uptodate and rebuild if it is not.
1780
1781 @param force: True if the registry needs rebuilding for sure.
1782 """
1783
1784 if force or self.rebuildNeeded():
1785 self.info("Rebuilding registry")
1786 if force:
1787 self.info("Rebuild of registry is forced")
1788 if self.rebuildNeeded():
1789 self.info("Rebuild of registry is needed")
1790 self.clean()
1791 mtime = self.seconds()
1792 for path in self._paths:
1793 if not self.addRegistryPath(path):
1794 self._parser.removeDirectoryByPath(path)
1795 self.mtime = mtime
1796 self.save(True)
1797
1799 return self._modmtime >= _getMTime(__file__)
1800
1801
1803
1804 - def __init__(self, fromRegistry=None, onlyBundles=None):
1805 """
1806 @param fromRegistry: The registry to subset, or the default.
1807 @type fromRegistry: L{ComponentRegistry}
1808 @param onlyBundles: If given, only include the subset of the
1809 registry that is provided by bundles whose names are in this
1810 list.
1811 @type onlyBundles: list of str
1812 """
1813 self.fromRegistry = fromRegistry
1814 self.onlyBundles = onlyBundles
1815
1816 - def dump(self, fd):
1833
1834 pred = lambda c: (filter(lambda f: fileIsBundled(c.getBase(),
1835 f.getFilename()),
1836 c.getFiles())
1837 or filter(lambda e: fileIsBundled(c.getBase(),
1838 e.getLocation()),
1839 c.getEntries()))
1840 components = filter(pred, reg.getComponents())
1841
1842 pred = lambda p: p.getEntry().getLocation() in bundledfiles
1843 plugs = filter(pred, reg.getPlugs())
1844
1845 directories = []
1846
1847 regwriter = RegistryWriter(components, plugs, bundles, directories)
1848 regwriter.dump(fd)
1849
1850 __registry = None
1851
1852
1854 """
1855 Make a bundle from a subset of all loaded modules, also writing out
1856 a registry file that can apply to that subset of the global
1857 registry. Suitable for use as a FLU_ATEXIT handler.
1858
1859 @param outfile: The path to which a zip file will be written.
1860 @type outfile: str
1861 @param outreg: The path to which a registry file will be written.
1862 @type outreg: str
1863 @param prefixes: A list of prefixes to which to limit the export. If
1864 not given, package up all modules. For example, "flumotion" would
1865 limit the output to modules that start with "flumotion".
1866 @type prefixes: list of str
1867 """
1868 from twisted.python import reflect
1869
1870 def getUsedModules(prefixes):
1871 ret = {}
1872 for modname in sys.modules:
1873 if prefixes and not filter(modname.startswith, prefixes):
1874 continue
1875 try:
1876 module = reflect.namedModule(modname)
1877 if hasattr(module, '__file__'):
1878 ret[modname] = module
1879 else:
1880 log.info('makebundle', 'Module %s has no file', module)
1881 except ImportError:
1882 log.info('makebundle', 'Could not import %s', modname)
1883 return ret
1884
1885 def calculateModuleBundleMap():
1886 allbundles = getRegistry().getBundles()
1887 ret = {}
1888 for bundle in allbundles:
1889 for directory in bundle.getDirectories():
1890 for bundleFile in directory.getFiles():
1891 path = os.path.join(directory.getName(),
1892 bundleFile.getLocation())
1893 parts = path.split(os.path.sep)
1894 if parts[-1].startswith('__init__.py'):
1895 parts.pop()
1896 elif parts[-1].endswith('.py'):
1897 parts[-1] = parts[-1][:-3]
1898 else:
1899
1900 continue
1901 modname = '.'.join(parts)
1902 ret[modname] = bundle
1903 return ret
1904
1905 def makeMergedBundler(modules, modulebundlemap):
1906 ret = MergedBundler()
1907 basket = getRegistry().makeBundlerBasket()
1908 for modname in modules:
1909 modfilename = modules[modname].__file__
1910 if modname in modulebundlemap:
1911 bundleName = modulebundlemap[modname].getName()
1912 for depBundleName in basket.getDependencies(bundleName):
1913 ret.addBundler(basket.getBundlerByName(depBundleName))
1914 else:
1915 if modfilename.endswith('.pyc'):
1916 modfilename = modfilename[:-1]
1917 if os.path.isdir(modfilename):
1918 with_init = os.path.join(modfilename, '__init__.py')
1919 if os.path.exists(with_init):
1920 modfilename = with_init
1921 nparts = len(modname.split('.'))
1922 if '__init__' in modfilename:
1923 nparts += 1
1924 relpath = os.path.join(*modfilename.split(
1925 os.path.sep)[-nparts:])
1926 ret.add(modfilename, relpath)
1927 return ret
1928
1929 modules = getUsedModules(prefixes)
1930 modulebundlemap = calculateModuleBundleMap()
1931 bundler = makeMergedBundler(modules, modulebundlemap)
1932
1933 print 'Writing bundle to', outfile
1934 open(outfile, 'w').write(bundler.bundle().getZip())
1935
1936 print 'Writing registry to', outreg
1937 bundlers_used = [b.name for b in bundler.getSubBundlers()]
1938 regwriter = RegistrySubsetWriter(onlyBundles=bundlers_used)
1939 regwriter.dump(open(outreg, 'w'))
1940
1941
1962