From ece3b0db476c1b2498877e9e035d1f3a2df9f48e Mon Sep 17 00:00:00 2001 From: Taeyeon Mori Date: Fri, 30 Jun 2017 16:37:50 +0900 Subject: [PATCH] Import flif --- .gitignore | 5 ++ README.md | 45 +++++++++++++ imglibs/__init__.py | 0 imglibs/filebuf.py | 47 +++++++++++++ imglibs/flif.py | 139 ++++++++++++++++++++++++++++++++++++++ libflif_build.py | 119 ++++++++++++++++++++++++++++++++ simplecpp.py | 160 ++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 515 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 imglibs/__init__.py create mode 100644 imglibs/filebuf.py create mode 100644 imglibs/flif.py create mode 100755 libflif_build.py create mode 100644 simplecpp.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e98ab5d --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*.c +*.o +*.so +__pycache__ + diff --git a/README.md b/README.md new file mode 100644 index 0000000..bae0e2c --- /dev/null +++ b/README.md @@ -0,0 +1,45 @@ +pyimglibs +========= + +Bindings for image format libraries based on cffi. + +Package contents +---------------- +Module | Description +------------------ | ----------------------------------- +`imglibs._libflif` | Free Lossless Image Format bindings +`imglibs._libwebp` | WebP bindings +`imglibs.flif` | PILLOW format for flif +`imglibs.webp` | (alternative) PILLOW format for webp +`imglibs.pillows` | Meta-module for importing all PILLOW formats + + +License +======= +You can use this code according to the MIT license, but beware that the +wrapped image format libraries may impose additional restrictions in practice + +MIT License +----------- +``` +Copyright (c) 2017 Taeyeon Mori + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` + diff --git a/imglibs/__init__.py b/imglibs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/imglibs/filebuf.py b/imglibs/filebuf.py new file mode 100644 index 0000000..abd70c0 --- /dev/null +++ b/imglibs/filebuf.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# Helper for getting buffers from file objects + +from __future__ import unicode_literals, division + +import mmap + + +class FileBuffer(object): + def __init__(self, fileobj): + self.file = fileobj + + try: + self.fileno = self.file.fileno() + except OSError: + self.fileno = -1 + + if self.fileno != -1:# and self.fd.seekable(): # Python 2.x doesn't have seekable() + # size + self.file.seek(0, 2) + self.size = self.file.tell() + self.buffer = mmap.mmap(self.fileno, self.size, access=mmap.ACCESS_READ) + self.type = "mmap" + elif hasattr(self.file, "getbuffer"): # BytesIO + self.buffer = self.file.getbuffer() + self.size = len(self.buffer) + self.type = "buffer" + else: + self.buffer = self.file.read() + self.size = len(self.buffer) + self.type = "bytes" + + def close(self): + if self.type == "mmap": + self.file.close() + elif self.type == "bytes": + del self.buffer + elif self.type == "buffer": + self.buffer = None + else: + raise RuntimeError("Unknown FileBuffer type %s" % self.type) + + def __enter__(self): + return self + + def __exit__(self, t, e, tb): + self.close() diff --git a/imglibs/flif.py b/imglibs/flif.py new file mode 100644 index 0000000..f67430e --- /dev/null +++ b/imglibs/flif.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 + +import mmap + +from PIL import Image, ImageFile + +from .filebuf import FileBuffer +from ._libflif import lib as libflif, ffi + + +# --------------------------------------------------------- +# Image file +def _accept(prefix): + return prefix[:4] == b"FLIF" + + +class FLImageFile(ImageFile.ImageFile): + format = "FLIF" + format_description = "Free Lossless Image Format" + + def _open(self): + header = self.fp.read(30) + info = libflif.flif_read_info_from_memory(header, 30) + try: + # Mode/Channels + channels = libflif.flif_info_get_nb_channels(info) + if channels == 1: + if libflif.flif_info_get_depth(info) == 1: + self.mode = "1" + else: + self.mode = "L" + elif channels == 3: + self.mode = "RGB" + elif channels == 4: + self.mode = "RGBA" + else: + raise ValueError("Cannot open FLIF image with %i channels" % channels) + + # Size + self.size = ( + libflif.flif_info_get_width(info), + libflif.flif_info_get_height(info) + ) + finally: + libflif.flif_destroy_info(info) + + self.tile = [("libflif", (0, 0) + self.size, 0, ())] + + +Image.register_open(FLImageFile.format, FLImageFile, _accept) + +Image.register_extension(FLImageFile.format, ".flif") + +Image.register_mime(FLImageFile.format, "image/flif") +Image.register_mime(FLImageFile.format, "application/x-flif") + + +# --------------------------------------------------------- +# Decoder +class LibFLIFDecoder(ImageFile.PyDecoder): + _pulls_fd = True + + def init(self, args): + self.context = libflif.flif_create_decoder() + + def _import_image(self, image, bufsize, mode, readinto_fn): + # FIXME: Don't copy shit around TWICE; Probably want to do it in C + buf = bytearray(bufsize) + readinto_fn(image, ffi.from_buffer(buf), bufsize) + self.set_as_raw(bytes(buf), mode) + + def decode(self, _): + # Figure out best way to get pointer to data in memory + + with FileBuffer(self.fd) as fb: + # Decode + libflif.flif_decoder_decode_memory(self.context, ffi.from_buffer(fb.buffer), fb.size) + + # Get image + image = libflif.flif_decoder_get_image(self.context, 0) + + # Convert to PIL representation + # flif_image_read_into_* defined in libflif_build.py + if self.mode in ("L", "1"): + self._import_image(image, self.state.xsize * self.state.ysize, "L", libflif.flif_image_read_into_GRAY8) + elif self.mode in ("RGB",): + self._import_image(image, self.state.xsize * self.state.ysize * 3 + self.state.xsize, "RGB", libflif.flif_image_read_into_RGB8) + elif self.mode in ("RGBA",): + self._import_image(image, self.state.xsize + self.state.ysize * 4, "RGBA", libflif.flif_image_read_into_RGBA8) + + return 0, 0 + + def cleanup(self): + libflif.flif_destroy_decoder(self.context) + + +Image.register_decoder("libflif", LibFLIFDecoder) + + +# --------------------------------------------------------- +# Writing +def _save(image, fp, filename): + lossy = image.encoderinfo.get("lossy", 0) + + # Create FLIF_IMAGE + # FIXME: Probably should do it in C + mode = image.mode + if mode == "L": + fimage = libflif.flif_import_image_GRAY(image.size[0], image.size[1], image.tobytes(), image.size[0]) + elif mode == "RGB": + fimage = libflif.flif_import_image_RGB(image.size[0], image.size[1], image.tobytes(), image.size[0] * 3) + elif mode == "RGBA": + fimage = libflif.flif_import_image_RGBA(image.size[0], image.size[1], image.tobytes(), image.size[0] * 4) + else: + raise IOError("cannot write mode %s as FLIF (yet)" % mode) + + if fimage == ffi.NULL: + raise RuntimeError("Could not create FLIF_IMAGE") + + # Set up encoder + context = libflif.flif_create_encoder() + + libflif.flif_encoder_set_lossy(context, lossy) + + libflif.flif_encoder_add_image_move(context, fimage) + + # Encode + out_buf = ffi.new("void **") + out_size = ffi.new("size_t *") + + res = libflif.flif_encoder_encode_memory(context, out_buf, out_size) + + fp.write(ffi.buffer(out_buf[0], out_size[0])) + + libflif.flif_destroy_encoder(context) + libflif.flif_free_memory(out_buf[0]) + + +Image.register_save(FLImageFile.format, _save) diff --git a/libflif_build.py b/libflif_build.py new file mode 100755 index 0000000..929e931 --- /dev/null +++ b/libflif_build.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# libflif cffi bindings +# (c) 2017 Taeyeon Mori + +import os +import re +import cffi + +from simplecpp import simple_cpp + +flif_headers = "/usr/include/FLIF" + +# -------------------------------------------------------------- +# helper source code +source = """ + +#include +#include + +/* +uint32_t flif_python_callback(uint32_t quality, int64_t bytes_read, uint8_t decode_over, void *user_data, void *context) +{ + PyObject *py_callback = user_data; + + if (!user_data) + { + PyErr_SetString(PyExc_ValueError, "user_data cannot be NULL for python callbacks"); + return 0; + } + + // TODO +} */ + +void flif_image_read_into_GRAY8(FLIF_IMAGE *image, void *buffer, size_t buffer_size) +{ + uint32_t rows = flif_image_get_height(image); + uint32_t columns = flif_image_get_width(image); + + for (uint32_t row = 0; row < rows && (row + 1) * columns <= buffer_size; ++row) + { + flif_image_read_row_GRAY8(image, row, buffer + row * columns, columns); + } +} + +void flif_image_read_into_RGBA8(FLIF_IMAGE *image, void *buffer, size_t buffer_size) +{ + uint32_t rows = flif_image_get_height(image); + uint32_t columns = flif_image_get_width(image); + + for (uint32_t row = 0; row < rows && (row + 1) * columns * 4 <= buffer_size; ++row) + { + flif_image_read_row_RGBA8(image, row, buffer + row * columns * 4, columns * 4); + } +} + +void flif_image_read_into_RGB8(FLIF_IMAGE *image, void *buffer, size_t buffer_size) +{ + uint32_t rows = flif_image_get_height(image); + uint32_t columns = flif_image_get_width(image); + uint32_t row; + + for (row = 0; row < rows && row * columns * 3 + columns * 4 <= buffer_size; ++row) + { + // Read RGBA8 + uint8_t *row_buf = (uint8_t *)buffer + row * columns * 3; + flif_image_read_row_RGBA8(image, row, row_buf, columns * 4); + // Remove Alpha channel + for (uint32_t col = 1; col < columns; ++col) + { + row_buf[col * 3] = row_buf[col * 4]; + row_buf[col * 3 + 1] = row_buf[col * 4 + 1]; + row_buf[col * 3 + 2] = row_buf[col * 4 + 2]; + } + } + + // Last row + if (row < rows && buffer_size >= (row + 1) * columns * 3) + { + uint8_t lastrow_buf[columns * 4]; + uint8_t *row_buf = (uint8_t *)buffer + row * columns * 3; + flif_image_read_row_RGBA8(image, row, lastrow_buf, columns * 4); + + for (uint32_t col = 0; col < columns; ++col) + { + row_buf[col * 3] = lastrow_buf[col * 4]; + row_buf[col * 3 + 1] = lastrow_buf[col * 4 + 1]; + row_buf[col * 3 + 2] = lastrow_buf[col * 4 + 2]; + } + } +} +""" + +# -------------------------------------------------------------- +# Initialize CFFI +libflif = cffi.FFI() +libflif.set_source("imglibs._libflif", source, libraries=["flif"]) + + +# -------------------------------------------------------------- +# Import libflif symbols +defs = {} + +for header in ("flif_common.h", "flif_dec.h", "flif_enc.h"): + with open(os.path.join(flif_headers, header), "r") as f: + libflif.cdef("".join(simple_cpp(f, defs))) + + +# -------------------------------------------------------------- +# Define above helper funcs +libflif.cdef(""" +void flif_image_read_into_GRAY8(FLIF_IMAGE *image, void *buffer, size_t buffer_size); +void flif_image_read_into_RGBA8(FLIF_IMAGE *image, void *buffer, size_t buffer_size); +void flif_image_read_into_RGB8(FLIF_IMAGE *image, void *buffer, size_t buffer_size); +""") + +# -------------------------------------------------------------- +# Build +if __name__ == "__main__": + libflif.compile(verbose=True) diff --git a/simplecpp.py b/simplecpp.py new file mode 100644 index 0000000..907e7d0 --- /dev/null +++ b/simplecpp.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +# Simple C PreProcessor for cffi +# (c) 2017 Taeyeon Mori + +import re +import functools + +def parse_macro(mac): + name = [] + arglist = None + body = [] + + it = iter(mac) + + for c in it: + if c == "(": + arglist = [] + break + elif c.isspace(): + break + else: + name.append(c) + + name = "".join(name).strip() + + if arglist is not None: + thisarg = [] + for c in it: + if c == ")": + if thisarg: + arglist.append("".join(thisarg).strip()) + break + elif c == ",": + if thisarg: + arglist.append("".join(thisarg).strip()) + thisarg.clear() + else: + thisarg.append(c) + + body = "".join(it) + + if arglist: + argrep = re.compile(r"\b(%s)\b" % "|".join(arglist)) + fn = lambda args: argrep.sub(lambda m: args[arglist.index(m.group(1))], body) + + return name + "()", fn + + return name, body.strip() + + +def_re = re.compile(r"\b\w+\b") + + +def def_sub(defs, m): + token = m.group(0) + if token in defs: + return " " + defs[token] + " " + else: + return token + +def make_fnmacrocall_pattern(definitions): + fns = [x[:-2] for x in definitions.keys() if x[-2:] == "()"] + if fns: + return re.compile(r"\b(%s)\(" % "|".join(fns)) + return None + + +def sub_macros(definitions, fnpattern, line): + if fnpattern: + m = fnpattern.search(line) + while m: + args = [] + bracket_level = 0 + current = [] + argslen = 1 + for c in line[m.end(1)+1:]: + argslen += 1 + if c == "," and bracket_level == 0: + args.append("".join(current)) + current.clear() + else: + if c in "([{": + bracket_level += 1 + elif c in ")]}": + bracket_level -= 1 + if bracket_level < 0: + if current: + args.append("".join(current)) + break + current.append(c) + + line = line[:m.start(1)] + definitions[m.group(1) + "()"](args) + line[m.end(1) + argslen:] + + m = fnpattern.search(line) + + return def_re.sub(functools.partial(def_sub, definitions), line) + + +def simple_cpp(lines, definitions=None): + """ Very simple C preprocessor """ + ifs = [] + definitions = definitions if definitions is not None else {} + fnpattern = make_fnmacrocall_pattern(definitions) + + prevline = None + + for line in lines: + # Continuation + if prevline: + line = prevline + line.strip() + prevline = None + if line.lstrip("\r\n").endswith("\\"): + prevline = line.lstrip("\r\n")[:-1] + continue + # comments + while "/*" in line and "*/" in line: + line = line[:line.index("/*")] + line[line.index("*/") + 2:] + if "//" in line: + line = line[:line.index("//")] + if line.strip(): + line += "\n" + if "/*" in line: + prevline = line.lstrip("\r\n") + continue + # Preprocessor directives + if line.rstrip().startswith("#"): + # Process + cpp_line = line.rstrip()[1:].strip().split(maxsplit=1) + directive = cpp_line[0] + args = cpp_line[1] if len(cpp_line) > 1 else None + + if directive == "ifdef": + ifs.append(args in definitions or args + "()" in definitions) + elif directive == "ifndef": + ifs.append(args not in definitions and args + "()" not in definitions) + elif directive == "if": + print("Simple CPP Warning: #if is unsupported: %s" % args) + ifs.append(False) + elif directive == "elif": + print("Simple CPP Warning: #elif is unsupported: %s" % args) + ifs[-1] = False + elif directive == "else": + ifs[-1] = not ifs[-1] + elif directive == "endif": + ifs.pop() + elif all(ifs): + if directive == "define": + name, value = parse_macro(args) + definitions[name] = value + if name[-2:] == "()": + fnpattern = make_fnmacrocall_pattern(definitions) + elif value: + yield "#define %s ...\n" % name + else: + print("Simple CPP Warning: Ignoring unknown directive %s" % ((directive, args),)) + elif all(ifs): + yield sub_macros(definitions, fnpattern, line) + + if prevline: + raise ValueError("Line continuation on last line!")