Import flif

.gitignore

@@ -0,0 +1,5 @@
+*.c
+*.o
+*.so
+__pycache__
+

README.md

@@ -0,0 +1,45 @@
+pyimglibs
+=========
+
+Bindings for image format libraries based on cffi.
+
+Package contents
+----------------
+Module             | Description
+------------------ | -----------------------------------
+`imglibs._libflif` | Free Lossless Image Format bindings
+`imglibs._libwebp` | WebP bindings
+`imglibs.flif`     | PILLOW format for flif
+`imglibs.webp`     | (alternative) PILLOW format for webp
+`imglibs.pillows`  | Meta-module for importing all PILLOW formats
+
+
+License
+=======
+You can use this code according to the MIT license, but beware that the
+wrapped image format libraries may impose additional restrictions in practice
+
+MIT License
+-----------
+```
+Copyright (c) 2017 Taeyeon Mori
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+```
+

imglibs/filebuf.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+# Helper for getting buffers from file objects
+
+from __future__ import unicode_literals, division
+
+import mmap
+
+
+class FileBuffer(object):
+    def __init__(self, fileobj):
+        self.file = fileobj
+
+        try:
+            self.fileno = self.file.fileno()
+        except OSError:
+            self.fileno = -1
+
+        if self.fileno != -1:# and self.fd.seekable(): # Python 2.x doesn't have seekable()
+            # size
+            self.file.seek(0, 2)
+            self.size = self.file.tell()
+            self.buffer = mmap.mmap(self.fileno, self.size, access=mmap.ACCESS_READ)
+            self.type = "mmap"
+        elif hasattr(self.file, "getbuffer"): # BytesIO
+            self.buffer = self.file.getbuffer()
+            self.size = len(self.buffer)
+            self.type = "buffer"
+        else:
+            self.buffer = self.file.read()
+            self.size = len(self.buffer)
+            self.type = "bytes"
+
+    def close(self):
+        if self.type == "mmap":
+            self.file.close()
+        elif self.type == "bytes":
+            del self.buffer
+        elif self.type == "buffer":
+            self.buffer = None
+        else:
+            raise RuntimeError("Unknown FileBuffer type %s" % self.type)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, t, e, tb):
+        self.close()

imglibs/flif.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+
+import mmap
+
+from PIL import Image, ImageFile
+
+from .filebuf import FileBuffer
+from ._libflif import lib as libflif, ffi
+
+
+# ---------------------------------------------------------
+# Image file
+def _accept(prefix):
+    return prefix[:4] == b"FLIF"
+
+
+class FLImageFile(ImageFile.ImageFile):
+    format = "FLIF"
+    format_description = "Free Lossless Image Format"
+
+    def _open(self):
+        header = self.fp.read(30)
+        info = libflif.flif_read_info_from_memory(header, 30)
+        try:
+            # Mode/Channels
+            channels = libflif.flif_info_get_nb_channels(info)
+            if channels == 1:
+                if libflif.flif_info_get_depth(info) == 1:
+                    self.mode = "1"
+                else:
+                    self.mode = "L"
+            elif channels == 3:
+                self.mode = "RGB"
+            elif channels == 4:
+                self.mode = "RGBA"
+            else:
+                raise ValueError("Cannot open FLIF image with %i channels" % channels)
+
+            # Size
+            self.size = (
+                libflif.flif_info_get_width(info),
+                libflif.flif_info_get_height(info)
+            )
+        finally:
+            libflif.flif_destroy_info(info)
+
+        self.tile = [("libflif", (0, 0) + self.size, 0, ())]
+
+
+Image.register_open(FLImageFile.format, FLImageFile, _accept)
+
+Image.register_extension(FLImageFile.format, ".flif")
+
+Image.register_mime(FLImageFile.format, "image/flif")
+Image.register_mime(FLImageFile.format, "application/x-flif")
+
+
+# ---------------------------------------------------------
+# Decoder
+class LibFLIFDecoder(ImageFile.PyDecoder):
+    _pulls_fd = True
+
+    def init(self, args):
+        self.context = libflif.flif_create_decoder()
+
+    def _import_image(self, image, bufsize, mode, readinto_fn):
+        # FIXME: Don't copy shit around TWICE; Probably want to do it in C
+        buf = bytearray(bufsize)
+        readinto_fn(image, ffi.from_buffer(buf), bufsize)
+        self.set_as_raw(bytes(buf), mode)
+
+    def decode(self, _):
+        # Figure out best way to get pointer to data in memory
+
+        with FileBuffer(self.fd) as fb:
+            # Decode
+            libflif.flif_decoder_decode_memory(self.context, ffi.from_buffer(fb.buffer), fb.size)
+
+        # Get image
+        image = libflif.flif_decoder_get_image(self.context, 0)
+
+        # Convert to PIL representation
+        # flif_image_read_into_* defined in libflif_build.py
+        if self.mode in ("L", "1"):
+            self._import_image(image, self.state.xsize * self.state.ysize, "L", libflif.flif_image_read_into_GRAY8)
+        elif self.mode in ("RGB",):
+            self._import_image(image, self.state.xsize * self.state.ysize * 3 + self.state.xsize, "RGB", libflif.flif_image_read_into_RGB8)
+        elif self.mode in ("RGBA",):
+            self._import_image(image, self.state.xsize + self.state.ysize * 4, "RGBA", libflif.flif_image_read_into_RGBA8)
+
+        return 0, 0
+
+    def cleanup(self):
+        libflif.flif_destroy_decoder(self.context)
+
+
+Image.register_decoder("libflif", LibFLIFDecoder)
+
+
+# ---------------------------------------------------------
+# Writing
+def _save(image, fp, filename):
+    lossy = image.encoderinfo.get("lossy", 0)
+
+    # Create FLIF_IMAGE
+    # FIXME: Probably should do it in C
+    mode = image.mode
+    if mode == "L":
+        fimage = libflif.flif_import_image_GRAY(image.size[0], image.size[1], image.tobytes(), image.size[0])
+    elif mode == "RGB":
+        fimage = libflif.flif_import_image_RGB(image.size[0], image.size[1], image.tobytes(), image.size[0] * 3)
+    elif mode == "RGBA":
+        fimage = libflif.flif_import_image_RGBA(image.size[0], image.size[1], image.tobytes(), image.size[0] * 4)
+    else:
+        raise IOError("cannot write mode %s as FLIF (yet)" % mode)
+
+    if fimage == ffi.NULL:
+        raise RuntimeError("Could not create FLIF_IMAGE")
+
+    # Set up encoder
+    context = libflif.flif_create_encoder()
+
+    libflif.flif_encoder_set_lossy(context, lossy)
+
+    libflif.flif_encoder_add_image_move(context, fimage)
+
+    # Encode
+    out_buf = ffi.new("void **")
+    out_size = ffi.new("size_t *")
+
+    res = libflif.flif_encoder_encode_memory(context, out_buf, out_size)
+
+    fp.write(ffi.buffer(out_buf[0], out_size[0]))
+
+    libflif.flif_destroy_encoder(context)
+    libflif.flif_free_memory(out_buf[0])
+
+
+Image.register_save(FLImageFile.format, _save)

libflif_build.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+# libflif cffi bindings
+# (c) 2017 Taeyeon Mori
+
+import os
+import re
+import cffi
+
+from simplecpp import simple_cpp
+
+flif_headers = "/usr/include/FLIF"
+
+# --------------------------------------------------------------
+# helper source code
+source = """
+
+#include <FLIF/flif.h>
+#include <Python.h>
+
+/*
+uint32_t flif_python_callback(uint32_t quality, int64_t bytes_read, uint8_t decode_over, void *user_data, void *context)
+{
+    PyObject *py_callback = user_data;
+    
+    if (!user_data)
+    {
+        PyErr_SetString(PyExc_ValueError, "user_data cannot be NULL for python callbacks");
+        return 0;
+    }
+    
+    // TODO
+} */
+
+void flif_image_read_into_GRAY8(FLIF_IMAGE *image, void *buffer, size_t buffer_size)
+{
+    uint32_t rows = flif_image_get_height(image);
+    uint32_t columns = flif_image_get_width(image);
+
+    for (uint32_t row = 0; row < rows && (row + 1) * columns <= buffer_size; ++row)
+    {
+        flif_image_read_row_GRAY8(image, row, buffer + row * columns, columns);
+    }
+}
+
+void flif_image_read_into_RGBA8(FLIF_IMAGE *image, void *buffer, size_t buffer_size)
+{
+    uint32_t rows = flif_image_get_height(image);
+    uint32_t columns = flif_image_get_width(image);
+
+    for (uint32_t row = 0; row < rows && (row + 1) * columns * 4 <= buffer_size; ++row)
+    {
+        flif_image_read_row_RGBA8(image, row, buffer + row * columns * 4, columns * 4);
+    }
+}
+
+void flif_image_read_into_RGB8(FLIF_IMAGE *image, void *buffer, size_t buffer_size)
+{
+    uint32_t rows = flif_image_get_height(image);
+    uint32_t columns = flif_image_get_width(image);
+    uint32_t row;
+
+    for (row = 0; row < rows && row * columns * 3 + columns * 4 <= buffer_size; ++row)
+    {
+        // Read RGBA8
+        uint8_t *row_buf = (uint8_t *)buffer + row * columns * 3;
+        flif_image_read_row_RGBA8(image, row, row_buf, columns * 4);
+        // Remove Alpha channel
+        for (uint32_t col = 1; col < columns; ++col)
+        {
+            row_buf[col * 3] = row_buf[col * 4];
+            row_buf[col * 3 + 1] = row_buf[col * 4 + 1];
+            row_buf[col * 3 + 2] = row_buf[col * 4 + 2];
+        }
+    }
+
+    // Last row
+    if (row < rows && buffer_size >= (row + 1) * columns * 3)
+    {
+        uint8_t lastrow_buf[columns * 4];
+        uint8_t *row_buf = (uint8_t *)buffer + row * columns * 3;
+        flif_image_read_row_RGBA8(image, row, lastrow_buf, columns * 4);
+        
+        for (uint32_t col = 0; col < columns; ++col)
+        {
+            row_buf[col * 3] = lastrow_buf[col * 4];
+            row_buf[col * 3 + 1] = lastrow_buf[col * 4 + 1];
+            row_buf[col * 3 + 2] = lastrow_buf[col * 4 + 2];
+        }
+    }
+}
+"""
+
+# --------------------------------------------------------------
+# Initialize CFFI
+libflif = cffi.FFI()
+libflif.set_source("imglibs._libflif", source, libraries=["flif"])
+
+
+# --------------------------------------------------------------
+# Import libflif symbols
+defs = {}
+
+for header in ("flif_common.h", "flif_dec.h", "flif_enc.h"):
+    with open(os.path.join(flif_headers, header), "r") as f:
+        libflif.cdef("".join(simple_cpp(f, defs)))
+
+
+# --------------------------------------------------------------
+# Define above helper funcs
+libflif.cdef("""
+void flif_image_read_into_GRAY8(FLIF_IMAGE *image, void *buffer, size_t buffer_size);
+void flif_image_read_into_RGBA8(FLIF_IMAGE *image, void *buffer, size_t buffer_size);
+void flif_image_read_into_RGB8(FLIF_IMAGE *image, void *buffer, size_t buffer_size);
+""")
+
+# --------------------------------------------------------------
+# Build
+if __name__ == "__main__":
+    libflif.compile(verbose=True)

simplecpp.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+# Simple C PreProcessor for cffi
+# (c) 2017 Taeyeon Mori
+
+import re
+import functools
+
+def parse_macro(mac):
+    name = []
+    arglist = None
+    body = []
+
+    it = iter(mac)
+
+    for c in it:
+        if c == "(":
+            arglist = []
+            break
+        elif c.isspace():
+            break
+        else:
+            name.append(c)
+
+    name = "".join(name).strip()
+
+    if arglist is not None:
+        thisarg = []
+        for c in it:
+            if c == ")":
+                if thisarg:
+                    arglist.append("".join(thisarg).strip())
+                break
+            elif c == ",":
+                if thisarg:
+                    arglist.append("".join(thisarg).strip())
+                thisarg.clear()
+            else:
+                thisarg.append(c)
+
+    body = "".join(it)
+
+    if arglist:
+        argrep = re.compile(r"\b(%s)\b" % "|".join(arglist))
+        fn = lambda args: argrep.sub(lambda m: args[arglist.index(m.group(1))], body)
+
+        return name + "()", fn
+
+    return name, body.strip()
+
+
+def_re = re.compile(r"\b\w+\b")
+
+
+def def_sub(defs, m):
+    token = m.group(0)
+    if token in defs:
+        return " " + defs[token] + " "
+    else:
+        return token
+
+def make_fnmacrocall_pattern(definitions):
+    fns = [x[:-2] for x in definitions.keys() if x[-2:] == "()"]
+    if fns:
+        return re.compile(r"\b(%s)\(" % "|".join(fns))
+    return None
+
+
+def sub_macros(definitions, fnpattern, line):
+    if fnpattern:
+        m = fnpattern.search(line)
+        while m:
+            args = []
+            bracket_level = 0
+            current = []
+            argslen = 1
+            for c in line[m.end(1)+1:]:
+                argslen += 1
+                if c == "," and bracket_level == 0:
+                    args.append("".join(current))
+                    current.clear()
+                else:
+                    if c in "([{":
+                        bracket_level += 1
+                    elif c in ")]}":
+                        bracket_level -= 1
+                        if bracket_level < 0:
+                            if current:
+                                args.append("".join(current))
+                            break
+                    current.append(c)
+
+            line = line[:m.start(1)] + definitions[m.group(1) + "()"](args) + line[m.end(1) + argslen:]
+
+            m = fnpattern.search(line)
+
+    return def_re.sub(functools.partial(def_sub, definitions), line)
+
+
+def simple_cpp(lines, definitions=None):
+    """ Very simple C preprocessor """
+    ifs = []
+    definitions = definitions if definitions is not None else {}
+    fnpattern = make_fnmacrocall_pattern(definitions)
+
+    prevline = None
+
+    for line in lines:
+        # Continuation
+        if prevline:
+            line = prevline + line.strip()
+        prevline = None
+        if line.lstrip("\r\n").endswith("\\"):
+            prevline = line.lstrip("\r\n")[:-1]
+            continue
+        # comments
+        while "/*" in line and "*/" in line:
+            line = line[:line.index("/*")] + line[line.index("*/") + 2:]
+        if "//" in line:
+            line = line[:line.index("//")]
+            if line.strip():
+                line += "\n"
+        if "/*" in line:
+            prevline = line.lstrip("\r\n")
+            continue
+        # Preprocessor directives
+        if line.rstrip().startswith("#"):
+            # Process
+            cpp_line = line.rstrip()[1:].strip().split(maxsplit=1)
+            directive = cpp_line[0]
+            args = cpp_line[1] if len(cpp_line) > 1 else None
+
+            if directive == "ifdef":
+                ifs.append(args in definitions or args + "()" in definitions)
+            elif directive == "ifndef":
+                ifs.append(args not in definitions and args + "()" not in definitions)
+            elif directive == "if":
+                print("Simple CPP Warning: #if is unsupported: %s" % args)
+                ifs.append(False)
+            elif directive == "elif":
+                print("Simple CPP Warning: #elif is unsupported: %s" % args)
+                ifs[-1] = False
+            elif directive == "else":
+                ifs[-1] = not ifs[-1]
+            elif directive == "endif":
+                ifs.pop()
+            elif all(ifs):
+                if directive == "define":
+                    name, value = parse_macro(args)
+                    definitions[name] = value
+                    if name[-2:] == "()":
+                        fnpattern = make_fnmacrocall_pattern(definitions)
+                    elif value:
+                        yield "#define %s ...\n" % name
+                else:
+                    print("Simple CPP Warning: Ignoring unknown directive %s" % ((directive, args),))
+        elif all(ifs):
+            yield sub_macros(definitions, fnpattern, line)
+
+    if prevline:
+        raise ValueError("Line continuation on last line!")