Merge branch 'develop'

README.md

@@ -17,7 +17,7 @@ pip install git+https://iffgit.fz-juelich.de/doc-utils/doc-utils
 After installing you can use the extended rst writers `rst2latex-extended` and `rst2html-extended` that are compatible
 with the standard versions but offer additional features:
 
-- Code blocks are formatted with the `listings` package (LaTeX only)
+- Code blocks are formatted with the `minted` package (LaTeX only)
 - Images can be embedded into the generated document (`embedded-image` directive, HTML only; for LaTeX this is already
   the default behavior)
 - Support for TikZ images by `tikz` and `tikz-figure` directives. TikZ code can be loaded from an external file or
@@ -31,4 +31,5 @@ with the standard versions but offer additional features:
   `--syntax-highlight=short` option with `rst2html` is recommended but long class names are also included in the this
   CSS file
 - `styles/rst2latex_fzj.sty`: Style with fzj colors and a custom title page
+- `styles/rst2luatex_fzj.sty`: Style with fzj colors and a custom title page for use with LuaTex / LuaLaTeX
 - `utils/Makefile`: Example Makefile for generating PDFs and HTML pages from Markdown and reStructuredText documents

docutils_extended/_version.py

-__version_info__ = (0, 1, 9)
+__version_info__ = (0, 2, 0)
 __version__ = '.'.join(map(str, __version_info__))

docutils_extended/directives/__init__.py

 from .code_directive import CodeBlock
 from .embedded_image_directive import EmbeddedImage
-from .tikz_directive import Tikz, TikzFigure, set_output_mode
+from .tikz_directive import Tikz, TikzFigure
+from .output_mode import get_output_mode, set_output_mode
 
 
-__all__ = ("CodeBlock", "EmbeddedImage", "Tikz", "TikzFigure", "set_output_mode")
+__all__ = ("CodeBlock", "EmbeddedImage", "Tikz", "TikzFigure", "get_output_mode", "set_output_mode")

docutils_extended/directives/code_directive.py

@@ -9,10 +9,8 @@ class CodeBlock(Directive):
     option_spec = {}
     has_content = True
 
-    rstlang_to_listingslang = {"pytb": "{}", "text": "{}"}
-
     def run(self):
-        language = self.rstlang_to_listingslang.get(self.arguments[0], self.arguments[0])
+        language = self.arguments[0]
         content = u"\n".join(self.content)
-        latex = u"\\begin{{lstlisting}}[language={}]\n{}\n\\end{{lstlisting}}".format(language, content)
+        latex = u"\\begin{{minted}}{{{}}}\n{}\n\\end{{minted}}".format(language, content)
         return [nodes.raw("", latex, format="latex")]

docutils_extended/directives/embedded_image_directive.py

+# -*- coding: utf-8 -*-
+
 import base64
 import requests
+import os
+import shutil
+import subprocess
+import tempfile
 import urlparse
 import sys
 from io import BytesIO
@@ -9,25 +15,82 @@ from docutils.parsers.rst import Directive
 from docutils.parsers.rst import directives, states
 from docutils.nodes import fully_normalize_name, whitespace_normalize_name
 from docutils.parsers.rst.roles import set_classes
+from .output_mode import get_output_mode
+
+
+TMP_IMAGE_FILENAME = "_downloaded_image{i:04d}.{ext}"
+OUT_PDF_FILENAME = "_converted_image{i:04d}.pdf"
 
 
 def is_url(url):
     return urlparse.urlparse(url).scheme not in ("", "file")
 
 
-def convert_image_to_pillow_image_and_html_base64(image_filepath_or_url):
-    if is_url(image_filepath_or_url):
+def load_and_convert_image(image_filepath_or_url, convert_to_base64=True, convert_svg_to_pdf=False):
+    self = load_and_convert_image
+    if not hasattr(self, "image_number"):
+        self.image_number = 1
+    image = None
+    image_content = None
+    image_filepath = None
+    image_is_pdf = None
+    image_is_svg = image_filepath_or_url.lower().endswith(".svg")
+    image_is_remote = is_url(image_filepath_or_url)
+    if image_is_svg and convert_svg_to_pdf and (image_is_remote or convert_to_base64):
+        tmp_dir = tempfile.mkdtemp()
+    else:
+        tmp_dir = None
+    if image_is_remote:
         image_url = image_filepath_or_url
         response = requests.get(image_url)
         response.raise_for_status()
         image_content = response.content
+        if not convert_to_base64 or (image_is_svg and convert_svg_to_pdf):
+            image_filepath = TMP_IMAGE_FILENAME.format(i=self.image_number, ext=os.path.splitext(image_url)[1])
+            if tmp_dir is not None:
+                image_filepath = os.path.join(tmp_dir, image_filepath)
+            with open(image_filepath.encode(sys.getfilesystemencoding()), "wb") as image_file:
+                image_file.write(image_content)
     else:
         image_filepath = image_filepath_or_url
-        with open(image_filepath.encode(sys.getfilesystemencoding()), "rb") as image_file:
-            image_content = image_file.read()
-    image = Image.open(BytesIO(image_content))
-    encoded_string = "data:image/{format};base64,".format(format=image.format.lower()) + base64.b64encode(image_content)
-    return image, encoded_string
+    try:
+        if image_is_svg and convert_svg_to_pdf:
+            image_content = None
+            out_pdf_filename = OUT_PDF_FILENAME.format(i=self.image_number)
+            if convert_to_base64:
+                out_pdf_filename = os.path.join(tmp_dir, out_pdf_filename)
+            with open(os.devnull, "w") as devnull:
+                subprocess.check_call(
+                    ["rsvg-convert", "-f", "pdf", "-o", out_pdf_filename, image_filepath],
+                    stdout=devnull,
+                    stderr=devnull,
+                )
+            image_filepath = out_pdf_filename
+            image_is_svg = False
+            image_is_pdf = True
+        if image_content is None:
+            with open(image_filepath.encode(sys.getfilesystemencoding()), "rb") as image_file:
+                image_content = image_file.read()
+    finally:
+        if tmp_dir is not None:
+            shutil.rmtree(tmp_dir)
+            tmp_dir = None
+    try:
+        image = Image.open(BytesIO(image_content))
+        image_format = image.format.lower()
+    except IOError:
+        if image_is_svg:
+            image_format = "svg+xml"
+        elif image_is_pdf:
+            image_format = "pdf"
+        else:
+            image_format = "unknown"
+    if convert_to_base64:
+        image_reference = "data:image/{format};base64,".format(format=image_format) + base64.b64encode(image_content)
+    else:
+        image_reference = image_filepath
+    self.image_number += 1
+    return image, image_reference
 
 
 class EmbeddedImage(Directive):
@@ -55,11 +118,15 @@ class EmbeddedImage(Directive):
                 'the "align" option.  Valid values for "align" are: "%s".'
                 % (self.name, self.options["align"], '", "'.join(self.align_values))
             )
+        is_output_mode_html = get_output_mode() == "html"
         messages = []
-        image, encoded_string = convert_image_to_pillow_image_and_html_base64(image_filepath)
-        reference = directives.uri(encoded_string)
+        image = None
+        image, image_reference = load_and_convert_image(
+            image_filepath, convert_to_base64=is_output_mode_html, convert_svg_to_pdf=not is_output_mode_html
+        )
+        reference = directives.uri(image_reference)
         self.options["uri"] = reference
-        if not any(attr in self.options for attr in ("height", "width", "scale")):
+        if not any(attr in self.options for attr in ("height", "width", "scale")) and image is not None:
             self.options["width"] = "{}px".format(image.size[0])
         reference_node = None
         if "target" in self.options:

docutils_extended/directives/output_mode.py

+# -*- coding: utf-8 -*-
+
+VALID_OUTPUT_MODES = ("latex", "html")
+_output_mode = "latex"
+
+
+class InvalidOutputModeError(Exception):
+    pass
+
+
+def get_output_mode():
+    return _output_mode
+
+
+def set_output_mode(output_mode):
+    global _output_mode
+
+    if output_mode in VALID_OUTPUT_MODES:
+        _output_mode = output_mode
+    else:
+        raise InvalidOutputModeError("{} is not valid output mode".format(output_mode))

docutils_extended/directives/tikz_directive.py

@@ -14,6 +14,7 @@ from docutils.parsers.rst import Directive
 from docutils.parsers.rst import directives, states
 from docutils.nodes import fully_normalize_name, whitespace_normalize_name
 from docutils.parsers.rst.roles import set_classes
+from .output_mode import get_output_mode
 
 
 TEMPLATE_LATEX = u"""
@@ -50,23 +51,6 @@ TMP_SVG_FILENAME = "tikz_picture.svg"
 OUT_SVG_FILENAME = "_tikz_rendered{:04d}.svg"
 
 
-VALID_OUTPUT_MODES = ("latex", "html")
-_output_mode = "latex"
-
-
-class InvalidOutputModeError(Exception):
-    pass
-
-
-def set_output_mode(output_mode):
-    global _output_mode
-
-    if output_mode in VALID_OUTPUT_MODES:
-        _output_mode = output_mode
-    else:
-        raise InvalidOutputModeError("{} is not valid output mode".format(output_mode))
-
-
 class TemporaryDirectory(object):
     def __init__(self):
         self.tmp_dir = tempfile.mkdtemp()
@@ -199,7 +183,7 @@ class Tikz(Directive):
             )
         if "document_options" not in self.options:
             self.options["document_options"] = "12pt, border=5pt"
-        is_output_mode_html = _output_mode == "html"
+        is_output_mode_html = get_output_mode() == "html"
         messages = []
         if read_from_file:
             with codecs.open(tikz_source_filename, "r", "utf-8") as f:

docutils_extended/writers/rst2latex_extended.py

@@ -19,8 +19,7 @@ except:
 
 from docutils.core import publish_cmdline
 from docutils.parsers.rst import directives
-from docutils.parsers.rst.directives.images import Image
-from ..directives import CodeBlock, Tikz, TikzFigure, set_output_mode
+from ..directives import CodeBlock, EmbeddedImage, Tikz, TikzFigure, set_output_mode
 
 
 description = (
@@ -37,7 +36,7 @@ def main():
     set_output_mode("latex")
     for directive_name in ("code", "code-block"):
         directives.register_directive(directive_name, CodeBlock)
-    directives.register_directive("embedded-image", Image)
+    directives.register_directive("embedded-image", EmbeddedImage)
     directives.register_directive("tikz", Tikz)
     directives.register_directive("tikz-figure", TikzFigure)
     publish_cmdline(writer_name="latex", description=description)

styles/rst2latex_fzj.sty

@@ -6,44 +6,30 @@
 \PassOptionsToPackage{warn}{textcomp}
 \usepackage{lmodern}
 \usepackage{amssymb}
-\usepackage{listings}
+\usepackage{minted}
 \usepackage{xcolor}
 \usepackage{iflang}
 \usepackage{tikz}
 \usetikzlibrary{calc}
 
 \definecolor{fzjblue}{RGB}{2,61,107}
+\definecolor{mintedbg}{RGB}{252, 252, 255}
 
 \renewcommand{\textbf}[1]{{\bfseries\color{fzjblue}#1}}
 
-% Based on the GitHub theme
-\lstset{
-    basicstyle=\ttfamily\color{black}\footnotesize,
-    keywordstyle=\color[RGB]{215, 58, 73},
-    emphstyle=\color[RGB]{0, 92, 197},
-    commentstyle=\color[RGB]{106, 115, 125},
-    stringstyle=\color[RGB]{0, 114, 0},
-    numbers=left,
-    numberstyle=\tiny\color{gray},
-    stepnumber=1,
-    tabsize=2,
-    breaklines,
-    breakatwhitespace,
-    showspaces=false,
-    showstringspaces=false,
-    backgroundcolor=\color[RGB]{252, 252, 255},
+\setminted{
+    bgcolor=mintedbg,
+    breaklines=true,
+    fontsize=\footnotesize,
     frame=single,
-    language=bash
+    linenos=true,
+    python3=true,
+    style=friendly,
+    tabsize=2
 }
-\lstset{literate=%
-    {Ö}{{\"O}}1
-    {Ä}{{\"A}}1
-    {Ü}{{\"U}}1
-    {ß}{{\ss}}1
-    {ü}{{\"u}}1
-    {ä}{{\"a}}1
-    {ö}{{\"o}}1
-    {~}{{\textasciitilde}}1
+% Set minted line number style
+\renewcommand{\theFancyVerbLine}{%
+    \sffamily\textcolor{gray}{\scriptsize\arabic{FancyVerbLine}}
 }
 
 \renewcommand{\maketitle}{

styles/rst2xetex_fzj.sty

+% Author: Ingo Heimbach
+% Contact: i.heimbach@fz-juelich.de
+%
+% Style for use with Docutils/rst2latex/lualatex.
+
+\PassOptionsToPackage{warn}{textcomp}
+\usepackage{fontspec}
+\usepackage{lmodern}
+\usepackage{amssymb}
+\usepackage{minted}
+\usepackage{ucharclasses}
+\usepackage{xcolor}
+\usepackage{iflang}
+\usepackage{tikz}
+\usetikzlibrary{calc}
+
+\newfontfamily\DejaVuSans{DejaVuSans}
+\newfontfamily\DejaVuSansMono{DejaVuSansMono}
+
+% define a fallback font for symbols (like emoticons) that are not available in DejaVu Sans Mono
+% solution taken from https://tex.stackexchange.com/a/32268
+\setTransitionsForMathematics{%
+    \makeatletter\let\curfamily\f@family\let\curshape\f@shape\let\curseries\f@series\makeatother\DejaVuSans%
+}{%
+    \fontfamily{\curfamily}\fontshape{\curshape}\fontseries{\curseries}\selectfont%
+}
+\setTransitionsForSymbols{%
+    \makeatletter\let\curfamily\f@family\let\curshape\f@shape\let\curseries\f@series\makeatother\DejaVuSans%
+}{%
+    \fontfamily{\curfamily}\fontshape{\curshape}\fontseries{\curseries}\selectfont%
+}
+
+\definecolor{fzjblue}{RGB}{2,61,107}
+\definecolor{mintedbg}{RGB}{252, 252, 255}
+
+\renewcommand{\textbf}[1]{{\bfseries\color{fzjblue}#1}}
+
+\setminted{
+    bgcolor=mintedbg,
+    breaklines=true,
+    formatcom=\DejaVuSansMono,
+    fontsize=\footnotesize,
+    frame=single,
+    linenos=true,
+    python3=true,
+    style=friendly,
+    tabsize=2
+}
+% Set minted line number style
+\renewcommand{\theFancyVerbLine}{%
+    \sffamily\textcolor{gray}{\scriptsize\arabic{FancyVerbLine}}
+}
+
+\renewcommand{\maketitle}{
+\providecommand{\documenttype}{\IfLanguageName{ngerman}{Technische Dokumentation}{Technical Documentation}}%
+\begin{titlepage}
+    \sffamily
+    \makeatletter
+    \begin{tikzpicture}[remember picture, overlay]
+        \node[anchor=north] at ($ (current page.center) + (0, 0.5\textheight) $) {%
+            \LARGE\scshape%
+            \parbox{\textwidth}{\centering\documenttype}%
+        };%
+        \fill[ultra thick, fill=fzjblue] ($ (current page.north west) - (0, 0.25\paperheight) $) rectangle +(\paperwidth, -0.1\paperheight);
+        \node at ($ (current page.north) - (0, 0.3\paperheight) $) {%
+            \LARGE\bfseries%
+            \parbox{\textwidth}{\centering \color{white}\@title}
+        };%
+        \node at ($ (current page.center) + (0, 0.05\textheight) $) {%
+            \Large%
+            \parbox{\textwidth}{\centering \@author}
+        };
+        \node at ($ (current page.center) + (0, -0.05\textheight) $) {%
+            \large%
+            \ifthenelse{\equal{\@date}{}} {
+                \today
+            }{
+                \@date
+            }
+
+        };
+        \node[anchor=south] at ($ (current page.center) + (0, -0.5\textheight) $) {%
+            \parbox{\textwidth}{\centering
+                Forschungszentrum Jülich GmbH \\[1ex]
+                Peter Grünberg Institut / Jülich Centre for Neutron Science \\
+                Technische Administration \\
+                (PGI/JCNS-TA)
+            }%
+        };%
+        \node[anchor=south west, inner sep=0, outer sep=0] at ($ (current page.south west) + (0.05\paperwidth, 1.0631) $) {%
+            \sffamily\color{fzjblue}\IfLanguageName{ngerman}{Mitglied der Helmholtz-Gemeinschaft}{Member of the Helmholtz Association}%
+        };
+        \node[anchor=south east, inner sep=0, outer sep=0] at (current page.south east) {
+            \includegraphics[height=0.1769\paperwidth]{fz_logo_schutzzone}%
+        };%
+    \end{tikzpicture}
+    \makeatother
+\end{titlepage}
+}

utils/Makefile

@@ -23,21 +23,21 @@ pdf: $(PDFFILES)
 	cp "$${TEMP_HTML_DIR}/$@" ./ && \
 	rm -rf "$${TEMP_HTML_DIR}"
 
-%.pdf: %.rst .rst2latex.sty .fz_logo_schutzzone.pdf
+%.pdf: %.rst .rst2xetex.sty .fz_logo_schutzzone.pdf
 	TEX_FILENAME="$$(basename $< .rst).tex" && \
 	TEMP_TEX_DIR="$$(mktemp -d)" && \
-	cp $< .rst2latex.sty "$${TEMP_TEX_DIR}/" && \
+	cp $< .rst2xetex.sty "$${TEMP_TEX_DIR}/" && \
 	cp .fz_logo_schutzzone.pdf "$${TEMP_TEX_DIR}/fz_logo_schutzzone.pdf" && \
 	pushd "$${TEMP_TEX_DIR}" && \
 	$(RST2LATEX) --documentclass=scrartcl \
 	             --hyperref-options=hidelinks \
-	             --stylesheet=.rst2latex.sty \
+	             --stylesheet=.rst2xetex.sty \
 	             --stylesheet-dirs="$$(pwd)" \
 	             --no-section-numbering \
 	             --use-latex-docinfo \
 	             --latex-preamble="\usepackage[ngerman]{babel}" \
 	             $< "$${TEX_FILENAME}" && \
-	latexmk -pdf -halt-on-error "$${TEX_FILENAME}" && \
+	latexmk -xelatex -halt-on-error -shell-escape "$${TEX_FILENAME}" && \
 	popd && \
 	mv "$${TEMP_TEX_DIR}/$@" ./ && \
 	rm -rf "$${TEMP_TEX_DIR}"
@@ -45,8 +45,8 @@ pdf: $(PDFFILES)
 %.rst: %.md
 	$(PANDOC) --from=markdown --to=rst -o $@ $<
 
-.rst2latex.sty:
-	curl -o $@ -L https://iffgit.fz-juelich.de/doc-utils/doc-utils/raw/master/styles/rst2latex_fzj.sty
+.rst2xetex.sty:
+	curl -o $@ -L https://iffgit.fz-juelich.de/doc-utils/doc-utils/raw/master/styles/rst2xetex_fzj.sty
 
 .rst2html.css:
 	curl -o $@ -L https://iffgit.fz-juelich.de/doc-utils/doc-utils/raw/master/styles/rst2html_githublike_fzj.css
@@ -55,6 +55,6 @@ pdf: $(PDFFILES)
 	curl -o $@ -L https://iffwww.iff.kfa-juelich.de/pub/downloads/fz_logo/fz_logo_schutzzone.pdf
 
 clean:
-	rm -f *.html *.pdf .rst2latex.sty .rst2html.css .fz_logo_schutzzone.pdf $(patsubst %.md,%.rst,$(MDFILES))
+	rm -f *.html *.pdf .rst2xetex.sty .rst2html.css .fz_logo_schutzzone.pdf $(patsubst %.md,%.rst,$(MDFILES))
 
 .PHONY: all clean html pdf