#!/usr/bin/env python
"""
@file    buildHTMLDocs.py
@author  Daniel Krajzewicz
@author  Michael Behrisch
@date    2011-10-20
@version $Id: buildHTMLDocs.py 20433 2016-04-13 08:00:14Z behrisch $

Converts wiki-documentation into HTML pages.

Determines what to convert, first: if a command line argument is given,
it is interpreted as the page to convert. Otherwise, "Special:AllPages" is
downloaded and parsed for obtaining the list of all pages which will be
converted in subsequent steps.

For each of the pages to convert, the HTML-representation of the
page is downloaded and stripped from wiki-header/footer, first.
Then, the image-links are extracted from the HTML page and stored 
temporarily, the links themselves are patched to point to local pages/images
(if the page behind the link exists).
The page is saved into options.mirror/<PAGE_PATH>. 

After parsing all pages, the images are downloaded and stored into 
options.mirror/images.

After downloading all data, the title page is extracted and the content
included in this page is extracted. This content is embedded into "index.html"
between the <!-- nav begins --> / <!-- nav ends --> markers. 
All pages downloaded earlier are loaded, and embedded into the index.html
between the <!-- content begins --> / <!-- content ends --> markers. Then,
the page is saved into options.output/<PAGE_PATH>. All images are copied
from options.mirror/images to options.output/images.

SUMO, Simulation of Urban MObility; see http://sumo.dlr.de/
Copyright (C) 2011-2016 DLR (http://www.dlr.de/) and contributors

This file is part of SUMO.
SUMO is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.
"""
from __future__ import absolute_import
from __future__ import print_function
import urllib
import os
import sys
import shutil
import datetime
from optparse import OptionParser

from mirrorWiki import readParsePage, readParseEditPage


def patchLinks(page, name):
    images = set()
    level = len(name.split("/")) - 1
    level = "../" * level
    b = page.find("<a href")
    while b >= 0:
        # images/files
        if page[b + 9:].startswith("/wiki/File:") or page[b + 9:].startswith("/wiki/Image:"):
            b2 = b
            b = page.find(":", b) + 1
            images.add(page[b2 + 9:page.find("\"", b)])
            page = page[:b2 + 9] + level + "images/" + page[b:]
        # pages (HTML)
        elif page[b + 9:].startswith("/wiki/"):
            e = b + 9 + 6
            e2 = page.find("\"", b + 9)
            link = page[e:e2]
            if link.find("action=edit") < 0:
                if link.find("#") > 0:
                    link = level + link.replace("#", ".html#")
                elif link.find("#") < 0 and not (link.endswith(".png") or link.endswith(".jpg") or link.endswith(".svg")):
                    link = level + link + ".html"
                page = page[:b + 9] + link + page[e2:]
            else:
                page = page[:b + 9] + "http://sourceforge.net/" + page[b + 10:]
        b = page.find("<a href", b + 1)
    return page, images, level


def patchImages(page, name):
    images = set()
    level = len(name.split("/")) - 1
    level = "../" * level
    b = page.find("<img ")
    b = page.find("src", b)
    while b >= 0:
        b += 5
        e = page.find("\"", b + 2)
        add = page[b:e]
        l = add[add.rfind("/"):]
        if add.find("thumb") >= 0:
            l = l[l.find("-") + 1:]
        images.add(add)
        page = page[:b] + level + "images/" + l + page[e:]
        b = page.find("<img", b + 1)
        b = page.find("src", b)
    page = page.replace(".svg.png", ".svg")
    return page, images


def parseWikiLink(l):
    if l.find("[[") >= 0:
        # internal link
        b = l.find("[") + 2
        e = l.find("]", b)
        t = l[b:e]
        if t.find("|") < 0:
            link = t
            text = t
        else:
            link, text = t.split("|")
        link = link.replace(" ", "_")
        if link.find("#") >= 0:
            link = link.replace("#", ".html#")
        else:
            link = link + ".html"
        # external link
    elif l.find("[") >= 0:
        b = l.find("[") + 1
        e = l.find("]", b)
        t = l[b:e]
        link = t[:t.find(" ")]
        text = t[t.find(" ") + 1:]
    else:
        # text
        text = l[l.find(" ") + 1:]
        link = ""
    return text, link

optParser = OptionParser()
optParser.add_option("-m", "--mirror", default="mirror", help="mirror folder")
optParser.add_option("-o", "--output", default="docs", help="output folder")
optParser.add_option("-i", "--index", default=os.path.join(os.path.dirname(
    __file__), "..", "..", "docs", "wiki", "index.html"), help="index template file")
optParser.add_option("-r", "--version", help="add version info")
(options, args) = optParser.parse_args()

try:
    os.mkdir(options.mirror)
except:
    pass
try:
    os.mkdir(options.mirror + "/images")
except:
    pass
images = set()
if len(args) == 0:
    p = readParsePage("Special:AllPages")
    p = p[p.find('<ul class="mw-allpages-chunk">'):]
    pages = p.split("<a ")
else:
    pages = ['href="/wiki/%s"' % a for a in args]
for p in pages:
    if not p.startswith("href"):
        continue
    b = p.find("/wiki/")
    e = p.find("\"", b)
    name = p[b + 6:e]
    if name.endswith(".css"):
        print("Skipping css-file %s" % name)
        continue
    print("Fetching %s" % name)
    c = readParsePage(name)
    if name.find("/") > 0:
        try:
            os.makedirs(os.path.join(options.mirror, name[:name.rfind("/")]))
        except:
            pass
    if True:  # name.find(".")<0:
        c, pi, level = patchLinks(c, name)
        for i in pi:
            images.add(i)
        c, pi = patchImages(c, name)
        for i in pi:
            images.add(i)
    name = name + ".html"
    fd = open(os.path.join(options.mirror, name), "w")
    fd.write(c)
    fd.close()

imageFiles = []
for i in images:
    print("Fetching image %s" % i)
    if i.find(":") >= 0:
        f = urllib.urlopen("http://sumo.dlr.de%s" % i)
        c = f.read()
        b = c.find("<div class=\"fullImageLink\" id=\"file\">")
        b = c.find("href=", b) + 6
        e = c.find("\"", b + 1)
        f = urllib.urlopen("http://sumo.dlr.de/%s" % c[b:e])
        i = i[i.find(":") + 1:]
    else:
        f = urllib.urlopen("http://sumo.dlr.de/%s" % i)
        i = i[i.rfind("/") + 1:]
    if i.find("px-") >= 0:
        i = i[:i.find('-') + 1]
    fd = open(os.path.join(options.mirror, "images", i), "wb")
    fd.write(f.read())
    fd.close()
    imageFiles.append(os.path.join("images", i))

# build navigation
nav = readParseEditPage("SUMO_User_Documentation")
lines = nav[nav.find("="):].split("\n")
level = 0
c = ""
hadHeader = False
for l in lines:
    if len(l) == 0:
        continue
    if l[0] == '=':
        text, link = parseWikiLink(" " + l.replace("=", ""))
        if hadHeader:
            if level > 0:
                c += "</ul></li>\n" * level
        spc = ' ' * (level + 1)
        c = c + spc + "<li>"
        if link != "":
            c = c + "<a href=\"" + link + "\">"
        c = c + text
        if link != "":
            c = c + "</a>"
        c = c + "</li>\n"
        hadHeader = True
        level = 0
        continue
    if l[0].find('*') < 0:
        continue
    text, link = parseWikiLink(l)
    nLevel = l.count('*')
    if level > nLevel:
        c = c + ("</ul></li>\n" * (level - nLevel))
    if level < nLevel:
        c = c + \
            ('<li style="list-style: none; display: inline"><ul>\n' *
             (nLevel - level))
    spc = ' ' * (nLevel + 1)
    #+ str(level) + "-" + str(nLevel)
    c = c + spc + "<li>"
    if link != "":
        c = c + '<a href="' + link + '">' + text + '</a>'
    else:
        c = c + text
    c = c + "</li>\n"
    level = nLevel
if level > 0:
    c += "</ul></li>\n" * level

# get template and embed navigation
fd = open(options.index)
tpl = fd.read()
fd.close()
b = tpl.find("<!-- nav begins -->")
b = tpl.find(">", b) + 1
e = tpl.find("<!-- nav ends -->")
tpl = tpl[:b] + c + tpl[e:]

# build HTML pages
try:
    os.mkdir(options.output)
except:
    pass
try:
    os.mkdir(options.output + "/images")
except:
    pass
for p in pages:
    if not p.startswith("href"):
        continue
    b = p.find("/wiki/")
    e = p.find("\"", b)
    name = p[b + 6:e]
    if name.endswith(".css"):
        print("Skipping css-file %s" % name)
        continue
    fromStr = 'generated on %s from <a href="http://sumo.dlr.de/wiki/%s">the wiki page for %s</a>' % (
        datetime.datetime.now(), name, name)
    name = name + ".html"
    t = os.path.join(options.output, name)
    fd = open(os.path.join(options.mirror, name))
    c = fd.read()
    if options.version:
        fromStr += " for SUMO %s" % options.version
    c = c.replace(
        '<div id="siteSub">From Sumo</div>', '<div id="siteSub">%s</div>' % fromStr)
    fd.close()
    #
    if name.find('/') >= 0:
        level = name.count("/")
    else:
        level = name.count("\\")
    level = "../" * level
    # title
    cc = tpl.replace("<title>", "<title>" + name[:name.rfind(".html")] + " - ")
    # css inclusion
    cc = cc.replace("sumo.css", level + "sumo.css")
    cc = cc.replace("logo.png", level + "logo.png")
    # links
    b = cc.find("<a href=")
    while b >= 0:
        b = cc.find('"', b)
        if not cc[b + 1:].startswith("http"):
            cc = cc[:b + 1] + level + cc[b + 1:]
        b = cc.find("<a href=", b)
    # content
    b = cc.find("<!-- content begins -->")
    e = cc.find("<!-- content ends -->", b)
    e = cc.find("<", e + 1) - 1
    cc = cc[:b] + c + cc[e:]

    try:
        os.makedirs(os.path.split(t)[0])
    except:
        pass
    fd = open(t, "w")
    fd.write(cc)
    fd.close()
for i in imageFiles:
    shutil.copy(
        os.path.join(options.mirror, i), os.path.join(options.output, i))
if os.path.exists(os.path.join(options.output, 'SUMO_User_Documentation.html')):
    shutil.copy(os.path.join(options.output, 'SUMO_User_Documentation.html'),
                os.path.join(options.output, 'index.html'))
