#!/usr/bin/env python
"""
@file    mirrorWiki.py
@author  Daniel Krajzewicz
@author  Michael Behrisch
@date    2011-10-20
@version $Id: mirrorWiki.py 20433 2016-04-13 08:00:14Z behrisch $

Mirrors wiki-documentation.

Determines what to mirror, first: if a command line argument is given,
it is interpreted as the page to mirror. Otherwise, "Special:AllPages" is
downloaded and parsed for obtaining the list of all pages which will be
converted in subsequent steps.

For each of the pages to mirror, the page is downloaded as for
being edited and is stripped from wiki-header/footer, first.
Then, the image-links are extracted from the HTML page and stored 
temporarily.
The page is saved into MIRROR_FOLDER/<PAGE_PATH>. 

After mirroring all pages, the images are downloaded and stored into 
MIRROR_FOLDER/images.

SUMO, Simulation of Urban MObility; see http://sumo.dlr.de/
Copyright (C) 2008-2016 DLR (http://www.dlr.de/) and contributors

This file is part of SUMO.
SUMO is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.
"""
from __future__ import absolute_import
from __future__ import print_function
import urllib
import os
import sys
import shutil
from optparse import OptionParser


def readParsePage(page):
    f = urllib.urlopen("http://sumo.dlr.de/wiki/%s" % page)
    c = f.read()
    b = c.find("This page was last modified on")
    e = c.find("<", b)
    lastMod = c[b:e]
    b = c.find('<a id="top"')
    e = c.find('<div class="printfooter">')
    c = c[b:e]
    c = c.replace('<h3 id="siteSub">From sumo</h3>', '')
    b = c.find('<div id="jump-to-nav"')
    e = c.find('</div>', b) + 6
    return c[:b] + c[e:] + '</div><hr/><div id="lastmod">' + lastMod + '</div>'


def readParseEditPage(page):
    f = urllib.urlopen(
        "http://sumo.dlr.de/w/index.php?title=%s&action=edit" % page)
    c = f.read()
    b = c.find("wpTextbox1")
    b = c.find('>', b) + 1
    e = c.find("</textarea>")
    return c[b:e]


def getImages(page):
    images = set()
    for t in ["Image:", "File:"]:
        b = page.find(t)
        while b >= 0:
            e = len(page)
            for ch in ["|", "\n", "]"]:
                pos = page.find(ch, b)
                if pos >= 0 and pos < e:
                    e = pos
            images.add(page[b:e].strip())
            b = page.find(t, b + 1)
    return images

if __name__ == "__main__":
    optParser = OptionParser()
    optParser.add_option(
        "-o", "--output", default="wiki", help="output folder")
    (options, args) = optParser.parse_args()

    try:
        os.makedirs(os.path.join(options.output, "images"))
    except:
        pass
    images = set()
    if len(args) == 0:
        p = readParsePage("Special:AllPages")
        p = p[p.find('<ul class="mw-allpages-chunk">'):]
        pages = p.split("<a ")
    else:
        pages = ['href="/wiki/%s"' % a for a in args]
    for p in pages:
        if not p.startswith("href"):
            continue
        b = p.find("/wiki/")
        e = p.find("\"", b)
        name = p[b + 6:e]
        print("Fetching %s" % name)
        c = readParseEditPage(name)
        if name.find("/") > 0:
            try:
                os.makedirs(
                    os.path.join(options.output, name[:name.rfind("/")]))
            except:
                pass
            images.update(getImages(c))
        name = name + ".txt"
        fd = open(os.path.join(options.output, name), "w")
        fd.write(c)
        fd.close()

    for i in images:
        print("Fetching image %s" % i)
        if i.find(":") >= 0:
            f = urllib.urlopen("http://sumo.dlr.de/wiki/%s" % i)
            c = f.read()
            b = c.find("<div class=\"fullImageLink\" id=\"file\">")
            b = c.find("href=", b) + 6
            e = c.find("\"", b + 1)
            f = urllib.urlopen("http://sourceforge.net/%s" % c[b:e])
            i = i[i.find(":") + 1:]
        else:
            f = urllib.urlopen("http://sourceforge.net/%s" % i)
            i = i[i.rfind("/") + 1:]
        if i.find("px-") >= 0:
            i = i[:i.find('-') + 1]
        fd = open(os.path.join(options.output, "images", i), "wb")
        fd.write(f.read())
        fd.close()
