1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """module for accessing mozilla xpi packages"""
23
24 from __future__ import generators
25 import zipfile
26 import os.path
27 from translate import __version__
28 import StringIO
29 import re
30
31
32
33 from translate.misc import zipfileext
34 ZipFileBase = zipfileext.ZipFileExt
35
36 from translate.misc import wStringIO
37
38
43
44 NamedStringInput = wStringIO.StringIO
45 NamedStringOutput = wStringIO.StringIO
46
48 def cp(a, b):
49 l = min(len(a), len(b))
50 for n in range(l):
51 if a[n] != b[n]:
52 return a[:n]
53 return a[:l]
54 if itemlist:
55 return reduce(cp, itemlist)
56 else:
57 return ''
58
60 def changed(*args, **kwargs):
61 self.changed = True
62 method(*args, **kwargs)
63 return changed
64
66 """catches output if there has been, before closing"""
76
78 """wrap the underlying close method, to pass the value to onclose before it goes"""
79 if self.changed:
80 value = self.getvalue()
81 self.onclose(value)
82 NamedStringInput.close(self)
83
85 """zip files call flush, not close, on file-like objects"""
86 value = self.getvalue()
87 self.onclose(value)
88 NamedStringInput.flush(self)
89
91 """use this method to force the closing of the stream if it isn't closed yet"""
92 if not self.closed:
93 self.close()
94
96 """a ZipFile that calls any methods its instructed to before closing (useful for catching stream output)"""
102
104 """remember to call the given method before closing"""
105 if hasattr(self, "pendingsaves"):
106 if not pendingsave in self.pendingsaves:
107 self.pendingsaves.append(pendingsave)
108 else:
109 self.pendingsaves = [pendingsave]
110
112 """close the stream, remembering to call any addcatcher methods first"""
113 if hasattr(self, "pendingsaves"):
114 for pendingsave in self.pendingsaves:
115 pendingsave()
116
117 if ZipFileCatcher is None:
118 self.oldclose()
119 else:
120 super(ZipFileCatcher, self).close()
121
123 """writes the string into the archive, overwriting the file if it exists..."""
124 if isinstance(zinfo_or_arcname, zipfile.ZipInfo):
125 filename = zinfo_or_arcname.filename
126 else:
127 filename = zinfo_or_arcname
128 if filename in self.NameToInfo:
129 self.delete(filename)
130 self.writestr(zinfo_or_arcname, bytes)
131 self.writeendrec()
132
135 """sets up the xpi file"""
136 self.includenonloc = kwargs.get("includenonloc", True)
137 if "includenonloc" in kwargs:
138 del kwargs["includenonloc"]
139 if "compression" not in kwargs:
140 kwargs["compression"] = zipfile.ZIP_DEFLATED
141 self.locale = kwargs.pop("locale", None)
142 self.region = kwargs.pop("region", None)
143 super(XpiFile, self).__init__(*args, **kwargs)
144 self.jarfiles = {}
145 self.findlangreg()
146 self.jarprefixes = self.findjarprefixes()
147 self.reverseprefixes = dict([
148 (prefix,jarfilename) for jarfilename, prefix in self.jarprefixes.iteritems() if prefix])
149 self.reverseprefixes["package/"] = None
150
152 """iterate through the jar files in the xpi as ZipFile objects"""
153 for filename in self.namelist():
154 if filename.lower().endswith('.jar'):
155 if filename not in self.jarfiles:
156 jarstream = self.openinputstream(None, filename)
157 jarfile = ZipFileCatcher(jarstream, mode=self.mode)
158 self.jarfiles[filename] = jarfile
159 else:
160 jarfile = self.jarfiles[filename]
161 yield filename, jarfile
162
164 """returns whether the given file is needed for localization (basically .dtd and .properties)"""
165 base, ext = os.path.splitext(filename)
166 return ext in (os.extsep + "dtd", os.extsep + "properties")
167
169 """finds the common prefix of all the files stored in the jar files"""
170 dirstructure = {}
171 locale = self.locale
172 region = self.region
173 localematch = re.compile("^[a-z]{2,3}(-[a-zA-Z]{2,3}|)$")
174 regionmatch = re.compile("^[a-zA-Z]{2,3}$")
175
176 osmatch = re.compile("^[a-z]{2,3}-(mac|unix|win)$")
177 for jarfilename, jarfile in self.iterjars():
178 jarname = "".join(jarfilename.split('/')[-1:]).replace(".jar", "", 1)
179 if localematch.match(jarname) and not osmatch.match(jarname):
180 if locale is None:
181 locale = jarname
182 elif locale != jarname:
183 locale = 0
184 elif regionmatch.match(jarname):
185 if region is None:
186 region = jarname
187 elif region != jarname:
188 region = 0
189 for filename in jarfile.namelist():
190 if filename.endswith('/'):
191 continue
192 if not self.islocfile(filename) and not self.includenonloc:
193 continue
194 parts = filename.split('/')[:-1]
195 treepoint = dirstructure
196 for partnum in range(len(parts)):
197 part = parts[partnum]
198 if part in treepoint:
199 treepoint = treepoint[part]
200 else:
201 treepoint[part] = {}
202 treepoint = treepoint[part]
203 localeentries = {}
204 if 'locale' in dirstructure:
205 for dirname in dirstructure['locale']:
206 localeentries[dirname] = 1
207 if localematch.match(dirname) and not osmatch.match(dirname):
208 if locale is None:
209 locale = dirname
210 elif locale != dirname:
211 print "locale dir mismatch - ", dirname, "but locale is", locale, "setting to 0"
212 locale = 0
213 elif regionmatch.match(dirname):
214 if region is None:
215 region = dirname
216 elif region != dirname:
217 region = 0
218 if locale and locale in localeentries:
219 del localeentries[locale]
220 if region and region in localeentries:
221 del localeentries[region]
222 if locale and not region:
223 if "-" in locale:
224 region = locale.split("-", 1)[1]
225 else:
226 region = ""
227 self.setlangreg(locale, region)
228
230 """set the locale and region of this xpi"""
231 if locale == 0 or locale is None:
232 raise ValueError("unable to determine locale")
233 self.locale = locale
234 self.region = region
235 self.dirmap = {}
236 if self.locale is not None:
237 self.dirmap[('locale', self.locale)] = ('lang-reg',)
238 if self.region:
239 self.dirmap[('locale', self.region)] = ('reg',)
240
242 """checks the uniqueness of the jar files contents"""
243 uniquenames = {}
244 jarprefixes = {}
245 for jarfilename, jarfile in self.iterjars():
246 jarprefixes[jarfilename] = ""
247 for filename in jarfile.namelist():
248 if filename.endswith('/'):
249 continue
250 if filename in uniquenames:
251 jarprefixes[jarfilename] = True
252 jarprefixes[uniquenames[filename]] = True
253 else:
254 uniquenames[filename] = jarfilename
255 for jarfilename, hasconflicts in jarprefixes.items():
256 if hasconflicts:
257 shortjarfilename = os.path.split(jarfilename)[1]
258 shortjarfilename = os.path.splitext(shortjarfilename)[0]
259 jarprefixes[jarfilename] = shortjarfilename+'/'
260
261 commonjarprefix = _commonprefix([prefix for prefix in jarprefixes.itervalues() if prefix])
262 if commonjarprefix:
263 for jarfilename, prefix in jarprefixes.items():
264 if prefix:
265 jarprefixes[jarfilename] = prefix.replace(commonjarprefix, '', 1)
266 return jarprefixes
267
269 """converts a zipfile filepath to an os-style filepath"""
270 return os.path.join(*zippath.split('/'))
271
273 """converts an os-style filepath to a zipfile filepath"""
274 return '/'.join(ospath.split(os.sep))
275
277 """uses a map to simplify the directory structure"""
278 parts = tuple(filename.split('/'))
279 possiblematch = None
280 for prefix, mapto in self.dirmap.iteritems():
281 if parts[:len(prefix)] == prefix:
282 if possiblematch is None or len(possiblematch[0]) < len(prefix):
283 possiblematch = prefix, mapto
284 if possiblematch is not None:
285 prefix, mapto = possiblematch
286 mapped = mapto + parts[len(prefix):]
287 return '/'.join(mapped)
288 return filename
289
291 """uses a map to rename files that occur straight in the xpi"""
292 if filename.startswith('bin/chrome/') and filename.endswith(".manifest"):
293 return 'bin/chrome/lang-reg.manifest'
294 return filename
295
297 """unmaps the filename..."""
298 possiblematch = None
299 parts = tuple(filename.split('/'))
300 for prefix, mapto in self.dirmap.iteritems():
301 if parts[:len(mapto)] == mapto:
302 if possiblematch is None or len(possiblematch[0]) < len(mapto):
303 possiblematch = (mapto, prefix)
304 if possiblematch is None:
305 return filename
306 mapto, prefix = possiblematch
307 reversemapped = prefix + parts[len(mapto):]
308 return '/'.join(reversemapped)
309
311 """uses a map to rename files that occur straight in the xpi"""
312 if filename == 'bin/chrome/lang-reg.manifest':
313 if self.locale:
314 return '/'.join(('bin', 'chrome', self.locale + '.manifest'))
315 else:
316 for otherfilename in self.namelist():
317 if otherfilename.startswith("bin/chrome/") and otherfilename.endswith(".manifest"):
318 return otherfilename
319 return filename
320
322 """converts a filename from within a jarfile to an os-style filepath"""
323 if jarfilename:
324 jarprefix = self.jarprefixes[jarfilename]
325 return self.ziptoospath(jarprefix+self.mapfilename(filename))
326 else:
327 return self.ziptoospath(os.path.join("package", self.mapxpifilename(filename)))
328
330 """converts an extracted os-style filepath to a jarfilename and filename"""
331 zipparts = ospath.split(os.sep)
332 prefix = zipparts[0] + '/'
333 if prefix in self.reverseprefixes:
334 jarfilename = self.reverseprefixes[prefix]
335 filename = self.reversemapfile('/'.join(zipparts[1:]))
336 if jarfilename is None:
337 filename = self.reversemapxpifilename(filename)
338 return jarfilename, filename
339 else:
340 filename = self.ostozippath(ospath)
341 if filename in self.namelist():
342 return None, filename
343 filename = self.reversemapfile('/'.join(zipparts))
344 possiblejarfilenames = [jarfilename for jarfilename, prefix in self.jarprefixes.iteritems() if not prefix]
345 for jarfilename in possiblejarfilenames:
346 jarfile = self.jarfiles[jarfilename]
347 if filename in jarfile.namelist():
348 return jarfilename, filename
349 raise IndexError("ospath not found in xpi file, could not guess location: %r" % ospath)
350
352 """checks whether the given file exists inside the xpi"""
353 if jarfilename is None:
354 return filename in self.namelist()
355 else:
356 jarfile = self.jarfiles[jarfilename]
357 return filename in jarfile.namelist()
358
360 """checks whether the given file exists inside the xpi"""
361 jarfilename, filename = self.ostojarpath(ospath)
362 if jarfilename is None:
363 return filename in self.namelist()
364 else:
365 jarfile = self.jarfiles[jarfilename]
366 return filename in jarfile.namelist()
367
375 inputstream = CatchPotentialOutput(contents, onclose)
376 self.addcatcher(inputstream.slam)
377 else:
378 jarfile = self.jarfiles[jarfilename]
379 contents = jarfile.read(filename)
380 inputstream = NamedStringInput(contents)
381 inputstream.name = self.jartoospath(jarfilename, filename)
382 if hasattr(self.fp, 'name'):
383 inputstream.name = "%s:%s" % (self.fp.name, inputstream.name)
384 return inputstream
385
387 """opens a file for writing (possibly inside a jarfile as a StringIO"""
388 if jarfilename is None:
389 def onclose(contents):
390 self.overwritestr(filename, contents)
391 else:
392 if jarfilename in self.jarfiles:
393 jarfile = self.jarfiles[jarfilename]
394 else:
395 jarstream = self.openoutputstream(None, jarfilename)
396 jarfile = ZipFileCatcher(jarstream, "w")
397 self.jarfiles[jarfilename] = jarfile
398 self.addcatcher(jarstream.slam)
399 def onclose(contents):
400 jarfile.overwritestr(filename, contents)
401 outputstream = wStringIO.CatchStringOutput(onclose)
402 outputstream.name = "%s %s" % (jarfilename, filename)
403 if jarfilename is None:
404 self.addcatcher(outputstream.slam)
405 else:
406 jarfile.addcatcher(outputstream.slam)
407 return outputstream
408
410 """Close the file, and for mode "w" and "a" write the ending records."""
411 for jarfile in self.jarfiles.itervalues():
412 jarfile.close()
413 super(XpiFile, self).close()
414
416 """test the xpi zipfile and all enclosed jar files..."""
417 for jarfile in self.jarfiles.itervalues():
418 jarfile.testzip()
419 super(XpiFile, self).testzip()
420
421 - def restructurejar(self, origjarfilename, newjarfilename, otherxpi, newlang, newregion):
422 """Create a new .jar file with the same contents as the given name, but rename directories, write to outputstream"""
423 jarfile = self.jarfiles[origjarfilename]
424 origlang = self.locale[:self.locale.find("-")]
425 if newregion:
426 newlocale = "%s-%s" % (newlang, newregion)
427 else:
428 newlocale = newlang
429 for filename in jarfile.namelist():
430 filenameparts = filename.split("/")
431 for i in range(len(filenameparts)):
432 part = filenameparts[i]
433 if part == origlang:
434 filenameparts[i] = newlang
435 elif part == self.locale:
436 filenameparts[i] = newlocale
437 elif part == self.region:
438 filenameparts[i] = newregion
439 newfilename = '/'.join(filenameparts)
440 fileoutputstream = otherxpi.openoutputstream(newjarfilename, newfilename)
441 fileinputstream = self.openinputstream(origjarfilename, filename)
442 fileoutputstream.write(fileinputstream.read())
443 fileinputstream.close()
444 fileoutputstream.close()
445
446 - def clone(self, newfilename, newmode=None, newlang=None, newregion=None):
447 """Create a new .xpi file with the same contents as this one..."""
448 other = XpiFile(newfilename, "w", locale=newlang, region=newregion)
449 origlang = self.locale[:self.locale.find("-")]
450
451 if newlang is None:
452 newlang = origlang
453 if newregion is None:
454 newregion = self.region
455 if newregion:
456 newlocale = "%s-%s" % (newlang, newregion)
457 else:
458 newlocale = newlang
459 for filename in self.namelist():
460 filenameparts = filename.split('/')
461 basename = filenameparts[-1]
462 if basename.startswith(self.locale):
463 newbasename = basename.replace(self.locale, newlocale)
464 elif basename.startswith(origlang):
465 newbasename = basename.replace(origlang, newlang)
466 elif basename.startswith(self.region):
467 newbasename = basename.replace(self.region, newregion)
468 else:
469 newbasename = basename
470 if newbasename != basename:
471 filenameparts[-1] = newbasename
472 renamefilename = "/".join(filenameparts)
473 print "cloning", filename, "and renaming to", renamefilename
474 else:
475 print "cloning", filename
476 renamefilename = filename
477 if filename.lower().endswith(".jar"):
478 self.restructurejar(filename, renamefilename, other, newlang, newregion)
479 else:
480 inputstream = self.openinputstream(None, filename)
481 outputstream = other.openoutputstream(None, renamefilename)
482 outputstream.write(inputstream.read())
483 inputstream.close()
484 outputstream.close()
485 other.close()
486 if newmode is None:
487 newmode = self.mode
488 if newmode == "w":
489 newmode = "a"
490 other = XpiFile(newfilename, newmode)
491 other.setlangreg(newlocale, newregion)
492 return other
493
495 """iterates through all the localization files with the common prefix stripped and a jarfile name added if neccessary"""
496 if includenonjars:
497 for filename in self.namelist():
498 if filename.endswith('/') and not includedirs:
499 continue
500 if not self.islocfile(filename) and not self.includenonloc:
501 continue
502 if not filename.lower().endswith(".jar"):
503 yield self.jartoospath(None, filename)
504 for jarfilename, jarfile in self.iterjars():
505 for filename in jarfile.namelist():
506 if filename.endswith('/'):
507 if not includedirs:
508 continue
509 if not self.islocfile(filename) and not self.includenonloc:
510 continue
511 yield self.jartoospath(jarfilename, filename)
512
513
515 """iterates through all the files. this is the method use by the converters"""
516 for inputpath in self.iterextractnames(includenonjars=True):
517 yield inputpath
518
520 """returns whether the given pathname exists in the archive"""
521 try:
522 jarfilename, filename = self.ostojarpath(fullpath)
523 except IndexError:
524 return False
525 return self.jarfileexists(jarfilename, filename)
526
531
533 """opens an output file given the full pathname"""
534 try:
535 jarfilename, filename = self.ostojarpath(fullpath)
536 except IndexError:
537 return None
538 return self.openoutputstream(jarfilename, filename)
539
540 if __name__ == '__main__':
541 import optparse
542 optparser = optparse.OptionParser(version="%prog "+__version__.sver)
543 optparser.usage = "%prog [-l|-x] [options] file.xpi"
544 optparser.add_option("-l", "--list", help="list files", \
545 action="store_true", dest="listfiles", default=False)
546 optparser.add_option("-p", "--prefix", help="show common prefix", \
547 action="store_true", dest="showprefix", default=False)
548 optparser.add_option("-x", "--extract", help="extract files", \
549 action="store_true", dest="extractfiles", default=False)
550 optparser.add_option("-d", "--extractdir", help="extract into EXTRACTDIR", \
551 default=".", metavar="EXTRACTDIR")
552 (options, args) = optparser.parse_args()
553 if len(args) < 1:
554 optparser.error("need at least one argument")
555 xpifile = XpiFile(args[0])
556 if options.showprefix:
557 for prefix, mapto in xpifile.dirmap.iteritems():
558 print "/".join(prefix), "->", "/".join(mapto)
559 if options.listfiles:
560 for name in xpifile.iterextractnames(includenonjars=True, includedirs=True):
561 print name
562 if options.extractfiles:
563 if options.extractdir and not os.path.isdir(options.extractdir):
564 os.mkdir(options.extractdir)
565 for name in xpifile.iterextractnames(includenonjars=True, includedirs=False):
566 abspath = os.path.join(options.extractdir, name)
567
568 currentpath = options.extractdir
569 subparts = os.path.dirname(name).split(os.sep)
570 for part in subparts:
571 currentpath = os.path.join(currentpath, part)
572 if not os.path.isdir(currentpath):
573 os.mkdir(currentpath)
574 outputstream = open(abspath, 'w')
575 jarfilename, filename = xpifile.ostojarpath(name)
576 inputstream = xpifile.openinputstream(jarfilename, filename)
577 outputstream.write(inputstream.read())
578 outputstream.close()
579