xconv/aav: Refine AdvancedAV

bin/xconv

@@ -23,16 +23,17 @@ xconv ffpmeg wrapper based on AdvancedAV
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 
-from advancedav import SimpleAV, version_info as aav_version_info
+from advancedav import version_info as aav_version_info
 
 from argparse import ArgumentParser, Action
 from abc import ABCMeta, abstractmethod
 from itertools import chain
 from functools import wraps
-from os.path import isdir, join as build_path, basename, splitext, exists
+from os.path import isdir, join as build_path, basename, dirname, splitext, exists, abspath
-from os import environ
+from os import environ, makedirs, mkdir
+
+version_info = 0, 1, 3
 
-version_info = 0, 1, 1
 
 # == Misc. helpers ==
 def __update(f, name, update):
@@ -50,6 +51,7 @@ def __defaults(obj, **defs):
 # == Profile Decorators ==
 profiles = {}
 
+
 def profile(f):
     """
     Define a XConv Profile
@@ -120,34 +122,22 @@ def singleaudio(profile):
     return wrapper
 
 
-# == Utilities for Profiles ==
+# == Extend AAV ==
-def downscale(outstream, width, height):
+import advancedav
-    # Scale while keeping aspect ratio; never upscale.
-    outstream.options["filter_complex"] = "scale=iw*min(1\,min(%i/iw\,%i/ih)):-1" % (width, height)
-
 
-def apply_defines(defines, object, *names, **maps):
-    # Apply defines to task/stream options
-    maps = (maps or {})
-    for name in names:
-        maps[name] = name
-    for define, option in maps.items():
-        if define in defines:
-            object.options[option] = defines[define]
 
-
+class OutputFile(advancedav.OutputFile):
-def options(object, **options):
+    def change_format(self, format=None, ext=None):
-    object.options.update(options)
+        # Diverge from decorated format.
-    return object
+        # Watch out for args.genout!!
+        if format:
+            self.container = format
+        if ext:
+            self.name = splitext(self.name)[0] + "." + ext
 
 
-def change_format(outfile, format=None, ext=None):
+class SimpleTask(advancedav.SimpleTask):
-    # Diverge from decorated format.
+    output_factory = OutputFile
-    # Watch out for args.genout!!
-    if format:
-        outfile.container = format
-    if ext:
-        outfile.name = splitext(outfile.name)[0] + "." + ext
 
 
 # == Profile definitions ==
@@ -157,19 +147,18 @@ def change_format(outfile, format=None, ext=None):
 def laptop(task):
     # add first video stream
     for s in task.iter_video_streams():
-        os = task.map_stream(s)
+        (task.map_stream(s)
-        options(os,
+            .set(codec="libx264",
-            codec="libx264",
+                tune=("fastdecode", "animation"),
-            tune=("fastdecode", "animation"),
+                profile="main",
-            profile="main",
+                preset="fast")
-            preset="fast")
+            .downscale(1280, 800))
-        downscale(os, 1280, 800)
         break
     # Add all audio streams (reencode to aac if necessary)
     for s in task.iter_audio_streams():
         os = task.map_stream(s)
         if s.codec != "aac":
-            os.options["codec"] = "aac"
+            os.set(codec="aac")
     # add all subtitle and attachment streams
     for s in chain(task.iter_subtitle_streams(), task.iter_attachment_streams()):
         task.map_stream(s)
@@ -183,10 +172,10 @@ def laptop(task):
 @singleaudio
 def flac(task, stream):
     if stream.codec == "ape":
-        stream.file.options["max_samples"] = "all" # Monkey's insane preset is insane.
+        stream.file.set(max_samples="all") # Monkey's insane preset is insane.
-    options(task.map_stream(stream),
+    (task.map_stream(stream)
-        codec="flac",
+        .set(codec="flac",
-        compression_level="10")
+            compression_level="10"))
     return True
 
 
@@ -198,15 +187,14 @@ def flac(task, stream):
 @features(argshax=None)
 @singleaudio
 def opus(task, stream, defines, args):
-    os = task.map_stream(stream)
+    os = (task.map_stream(stream)
-    options(os,
+        .set(codec="libopus",
-        codec="libopus",
+            vbr="on")
-        vbr="on")
+    # Defines
-    # options
+        .apply(defines, bitrate="b"))
-    apply_defines(defines, os, bitrate="b")
     # Output format
     if "ogg" in defines:
-        change_format(task.output, "ogg", "opus" if args.genout else None)
+        task.change_format("ogg", "opus" if args.genout else None)
     return True
 
 
@@ -216,18 +204,15 @@ def opus(task, stream, defines, args):
 @defines(stereo="Keep stereo channels")
 @singleaudio
 def audiobook(task, stream, defines):
-    out = task.map_stream(stream)
+    out = (task.map_stream(stream)
-    options(out,
+        .set(codec="libopus",
-        codec="libopus",
+            vbr="on",
-        vbr="on",
+            b="32k",
-        b="24k",
+            ac="1",
-        ac="1",
+            application="voip"))
-        application="voip",
-        frame_duration="40")
     if "stereo" in defines:
-        options(out,
+        out.set(ac="2",
-            ac="2",
+                b="36k")
-            b="32k")
     return True
 
 
@@ -237,11 +222,10 @@ def audiobook(task, stream, defines):
     fext="File extension")
 @features(argshax=None)
 def remux(task, defines, args):
-    change_format(task.output,
+    task.change_format(
         defines["format"] if "format" in defines else None,
         defines["fext"] if "fext" in defines and args.genout else None)
-    all(task.map_stream(s) for s in task.iter_streams())
+    return all(task.map_stream(s) for s in task.iter_streams())
-    return True
 
 
 # == Support code ==
@@ -298,8 +282,12 @@ def parse_args(argv):
     files = parser.add_argument_group("Files")
     files.add_argument("inputs", nargs="+", help="The input file(s)")
     files.add_argument("output", help="The output filename or directory")
-    files.add_argument("-m", "--merge", help="Merge streams from all inputs instead of mapping each input to an output", action="store_true")
+    multimode = files.add_mutually_exclusive_group()
+    multimode.add_argument("-m", "--merge", help="Merge streams from all inputs instead of mapping each input to an output", action="store_true")
+    multimode.add_argument("-C", "--concat", help="Concatenate streams from inputs instead of mapping", action="store_true")
     files.add_argument("-u", "--update", help="Only work on files that don't already exist", action="store_true")
+    files.add_argument("-c", "--create-parents", help="Create containing folders if they don't exist", action="store_true")
+    files.add_argument("-cc", "--create-folder", help="Create the output folder if it doesn't exist", action="store_true")
     profile = parser.add_argument_group("Profile")
     profile.add_argument("-p", "--profile", choices=profiles.keys(), required=True, help="Specify the profile. See the source code.")
     profile.add_argument("-l", "--list-profiles", action=ProfilesAction, nargs=0, help="List profiles and exit")
@@ -331,45 +319,69 @@ def main(argv):
 
     profile = profiles[args.profile]
 
+    if (args.create_parents or args.create_folder) and not isdir(dirname(args.output)):
+        makedirs(dirname(args.output))
+    if args.create_folder and not isdir(args.output):
+        mkdir(args.output)
+
     args.genout = isdir(args.output)
-    if not args.genout:
-        if len(args.inputs) > 1:
-            print("Output path '%s' is not a directory." % args.output)
-            return -1
 
     print("\033[36mXConv %s (AdvancedAV %s) (c) Taeyeon Mori\033[0m" % (".".join(map(str, version_info)), ".".join(map(str, aav_version_info))))
     print("\033[34mProfile: %s\033[0m" % args.profile)
 
     # Initialize AAV
-    aav = SimpleAV(ffmpeg=args.ffmpeg, ffprobe=args.ffprobe)
+    aav = advancedav.SimpleAV(ffmpeg=args.ffmpeg, ffprobe=args.ffprobe)
 
     if args.quiet:
         aav.global_conv_args = "-loglevel", "warning"
 
+    aav.global_args += "-hide_banner",
+
     # Collect Tasks
     tasks = []
 
     print("\033[35mCollecting Tasks..\033[0m")
 
     if args.merge:
-        task = aav.create_job(make_outfile(args, profile, args.inputs[0]))
+        task = SimpleTask(aav, make_outfile(args, profile, args.inputs[0]), profile.container)
 
         for input in args.inputs:
             task.add_input(input)
 
         tasks.append(task)
 
+    elif args.concat:
+        import tempfile, os
+        tmp = tempfile.NamedTemporaryFile(mode="w", delete=False)
+
+        with tmp:
+            tmp.write("ffconcat version 1.0\n")
+            tmp.write("# XConv concat file\n")
+            for f in map(abspath, args.inputs):
+                print("\033[36m  Concatenating %s\033[0m" % basename(f))
+                tmp.write("file '%s'\n" % f)
+
+        task = SimpleTask(aav, make_outfile(args, profile, args.inputs[0]), profile.container)
+
+        options(task.add_input(tmp.name),
+            f="concat",
+            safe="0")
+
+        tasks.append(task)
+
     else:
+        if not args.genout and len(args.inputs) > 1:
+            print("\033[31mOutput path '%s' is not a directory.\033[0m" % args.output)
+            return -1
+
         for input in args.inputs:
             out = make_outfile(args, profile, input)
             if args.update and exists(out):
                 continue
-            task = aav.create_job(out, profile.container)
+            task = SimpleTask(aav, out, profile.container)
             task.add_input(input)
             tasks.append(task)
 
-            task.options["hide_banner"] = None
-
     print("\033[35mPreparing Tasks..\033[0m")
 
     # Prepare profile parameters
@@ -395,6 +407,10 @@ def main(argv):
         print("\033[32m  Processing '%s'\033[0m" % basename(task.name))
         task.commit()
 
+    # Clean up
+    if args.concat:
+        os.unlink(tmp.name)
+
     return 0
 
 

lib/python/advancedav.py

@@ -33,7 +33,7 @@ from collections.abc import Iterable, Mapping, Sequence, Iterator, MutableMappin
 
 __all__ = "AdvancedAVError", "AdvancedAV", "SimpleAV"
 
-version_info = 2, 0, 1
+version_info = 2, 1, 0
 
 # Constants
 DEFAULT_CONTAINER = "matroska"
@@ -46,23 +46,57 @@ S_DATA = "d"
 S_UNKNOWN = "u"
 
 
-def stream_type(type_: str) -> str:
+# == Exceptions ==
-    """ Convert the ff-/avprobe type output to the notation used on the ffmpeg/avconv commandline """
+class AdvancedAVError(Exception):
-    lookup = {
+    pass
-        "Audio": S_AUDIO,
-        "Video": S_VIDEO,
-        "Subtitle": S_SUBTITLE,
-        "Attachment": S_ATTACHMENT,
-        "Data": S_DATA
-    }
 
-    return lookup.get(type_, S_UNKNOWN)
 
+# == Base Classes ==
+class ObjectWithOptions:
+    __slots__ = ()
 
-class AdvancedAVError(Exception):
+    def __init__(self, *, options=None, **more):
-    pass
+        super().__init__(**more)
+        self.options = options or {}
+
+    def apply(self, source, *names, **omap):
+        for name in names:
+            if name in source:
+                self.options[name] = source[name]
+        if omap:
+            for define, option in omap.items():
+                if define in source:
+                    self.options[option] = source[define]
+        return self
+
+    def set(self, **options):
+        self.options.update(options)
+        return self
 
 
+class ObjectWithMetadata:
+    __slots__ = ()
+
+    def __init__(self, *, metadata=None, **more):
+        super().__init__(**more)
+        self.metadata = metadata or {}
+
+    def apply_meta(self, source, *names, **mmap):
+        for name in names:
+            if name in source:
+                self.metadata[name] = source[name]
+        if mmap:
+            for name, key in mmap.items():
+                if name in source:
+                    self.metadata[key] = source[name]
+        return self
+
+    def meta(self, **metadata):
+        self.metadata.update(metadata)
+        return self
+
+
+# === Stream Classes ===
 class Stream:
     """
     Abstract stream base class
@@ -71,16 +105,17 @@ class Stream:
     """
     __slots__ = "file", "type", "index", "pertype_index", "codec", "profile"
 
-    def __init__(self, file: "File", type_: str, index: int=None, pertype_index: int=None,
+    def __init__(self, file: "File", type: str, index: int=None, pertype_index: int=None,
-                 codec: str=None, profile: str=None):
+                 codec: str=None, profile: str=None, **more):
+        super().__init__(**more)
         self.file = file
-        self.type = type_
+        self.type = type
         self.index = index
         self.pertype_index = pertype_index
         self.codec = codec
         self.profile = profile
 
-    def update_indices(self, index: int, pertype_index: int=None):
+    def _update_indices(self, index: int, pertype_index: int=None):
         """ Update the Stream indices """
         self.index = index
         if pertype_index is not None:
@@ -110,18 +145,18 @@ class InputStream(Stream):
         self.file = file
         self.language = language
 
-    def update_indices(self, index: int, pertype_index: int=None):
+    def _update_indices(self, index: int, pertype_index: int=None):
         """ InputStreams should not have their indices changed. """
         if index != self.index:
             raise ValueError("Cannot update indices on InputStreams! (This might mean there are bogus ids in the input")
         # pertype_index gets updated by File._add_stream() so we don't throw up if it gets updated
 
 
-class OutputStream(Stream):
+class OutputStream(Stream, ObjectWithOptions, ObjectWithMetadata):
     """
     Holds information about a mapped output stream
     """
-    __slots__ = "source", "metadata", "options"
+    __slots__ = "source", "options", "metadata"
 
     # TODO: support other parameters like frame resolution
 
@@ -131,28 +166,32 @@ class OutputStream(Stream):
 
     def __init__(self, file: "OutputFile", source: InputStream, stream_id: int, stream_pertype_id: int=None,
                  codec: str=None, options: Mapping=None, metadata: MutableMapping=None):
-        super().__init__(file, source.type, stream_id, stream_pertype_id, codec)
+        super().__init__(file=file, type=source.type, index=stream_id, pertype_index=stream_pertype_id,
+            codec=codec, options=options, metadata=metadata)
         self.source = source
-        self.options = options if options is not None else {}
-        self.metadata = metadata if metadata is not None else {}
 
-    # -- Manage Stream Metadata
-    def set_meta(self, key: str, value: str):
-        """ Set a metadata key """
-        self.metadata[key] = value
 
-    def get_meta(self, key: str) -> str:
+class OutputVideoStream(OutputStream):
-        """ Retrieve a metadata key """
+    def downscale(self, width, height):
-        return self.metadata[key]
+        # Scale while keeping aspect ratio; never upscale.
+        self.options["filter_complex"] = "scale=iw*min(1\,min(%i/iw\,%i/ih)):-1" % (width, height)
+        return self
 
 
-class File:
+def output_stream_factory(file, source, *args, **more):
+    return (OutputVideoStream if source.type == S_VIDEO else OutputStream)(file, source, *args, **more)
+
+
+# === File Classes ===
+class File(ObjectWithOptions):
     """
     ABC for Input- and Output-Files
     """
-    __slots__ = "name", "options", "_streams", "_streams_by_type"
+    __slots__ = "name", "_streams", "_streams_by_type", "options"
+
+    def __init__(self, name: str, options: dict=None, **more):
+        super().__init__(options=options, **more)
 
-    def __init__(self, name: str, options: dict=None):
         self.name = name
 
         self.options = options if options is not None else {}
@@ -166,7 +205,7 @@ class File:
 
     def _add_stream(self, stream: Stream):
         """ Add a stream """
-        stream.update_indices(len(self._streams), len(self._streams_by_type[stream.type]))
+        stream._update_indices(len(self._streams), len(self._streams_by_type[stream.type]))
         self._streams.append(stream)
         self._streams_by_type[stream.type].append(stream)
 
@@ -230,11 +269,16 @@ class File:
 class InputFile(File):
     """
     Holds information about an input file
+
+    :note: Modifying the options after accessing the streams results in undefined
+            behaviour! (Currently: Changes will only apply to conv call)
     """
     __slots__ = "pp", "_streams_initialized"
 
+    stream_factory = InputStream
+
     def __init__(self, pp: "AdvancedAV", filename: str, options: Mapping=None):
-        super().__init__(filename, dict(options.items()) if options else None)
+        super().__init__(name=filename, options=dict(options.items()) if options else None)
 
         self.pp = pp
 
@@ -246,6 +290,19 @@ class InputFile(File):
         r"(?:\s+\((?P<profile>[^\)]+)\))?(?:\s+(?P<extra>.+))?"
     )
 
+    @staticmethod
+    def _stream_type(type_: str) -> str:
+        """ Convert the ff-/avprobe type output to the notation used on the ffmpeg/avconv commandline """
+        lookup = {
+            "Audio": S_AUDIO,
+            "Video": S_VIDEO,
+            "Subtitle": S_SUBTITLE,
+            "Attachment": S_ATTACHMENT,
+            "Data": S_DATA
+        }
+
+        return lookup.get(type_, S_UNKNOWN)
+
     def _initialize_streams(self, probe: str=None) -> Iterator:
         """ Parse the ffprobe output
 
@@ -254,15 +311,18 @@ class InputFile(File):
         :rtype: Iterator[InputStream]
         """
         if probe is None:
-            probe = self.pp.call_probe(("-i", self.name))
+            if self.options:
+                probe = self.pp.call_probe(itertools.chain(Task.argv_options(self.options), ("-i", self.name)))
+            else:
+                probe = self.pp.call_probe(("-i", self.name))
 
         for match in self._reg_probe_streams.finditer(probe):
-            self._add_stream(InputStream(self,
+            self._add_stream(self.stream_factory(self,
-                                         stream_type(match.group("type")),
+                                                 self._stream_type(match.group("type")),
-                                         int(match.group("id")),
+                                                 int(match.group("id")),
-                                         match.group("lang"),
+                                                 match.group("lang"),
-                                         match.group("codec"),
+                                                 match.group("codec"),
-                                         match.group("profile")))
+                                                 match.group("profile")))
         self._streams_initialized = True
 
     @property
@@ -305,27 +365,44 @@ class InputFile(File):
             self._initialize_streams()
         return self._streams_by_type[S_SUBTITLE]
 
+    @property
+    def attachment_streams(self) -> Sequence:
+        """ All attachment streams (i.e. Fonts)
+
+        :rtype: Sequence[InputStream]
+        """
+        if not self._streams_initialized:
+            self._initialize_streams()
+        return self._streams_by_type[S_ATTACHMENT]
+
+    @property
+    def data_streams(self) -> Sequence:
+        """ All data streams
+
+        :rtype: Sequence[InputStream]
+        """
+        if not self._streams_initialized:
+            self._initialize_streams()
+        return self._streams_by_type[S_DATA]
 
-class OutputFile(File):
+
+class OutputFile(File, ObjectWithMetadata):
     """
     Holds information about an output file
     """
-    __slots__ = "task", "container", "metadata", "_mapped_sources"
+    __slots__ = "task", "container", "_mapped_sources", "metadata"
 
-    def __init__(self, task: "Task", name: str, container=DEFAULT_CONTAINER, options: Mapping=None):
+    stream_factory = staticmethod(output_stream_factory)
-        # Set default options
-        _options = {"c": "copy"}
 
-        if options is not None:
+    def __init__(self, task: "Task", name: str, container=DEFAULT_CONTAINER,
-            _options.update(options)
+            options: Mapping=None, metadata: Mapping=None):
+        super().__init__(name, options=options, metadata=metadata)
 
-        # Contstuct
+        self.options.setdefault("c", "copy")
-        super().__init__(name, _options)
 
         self.task = task
 
         self.container = container
-        self.metadata = {}
         """ :type: dict[str, str] """
 
         self._mapped_sources = set()
@@ -338,7 +415,7 @@ class OutputFile(File):
         map_stream() needs to ensure that the file the stream originates from is registered as input to this Task.
         However, when called repeatedly on streams of the same file, that is superflous.
         """
-        out = OutputStream(self, stream, -1, -1, codec, options)
+        out = self.stream_factory(self, stream, -1, -1, codec, options)
 
         self._add_stream(out)
         self._mapped_sources.add(stream)
@@ -413,17 +490,12 @@ class OutputFile(File):
         for stream in itertools.chain(self.video_streams,
                                       self.audio_streams,
                                       self.subtitle_streams):
-            stream.update_indices(len(self._streams))
+            stream._update_indices(len(self._streams))
             self._streams.append(stream)
-
+        return self
-    # -- Manage Global Metadata
-    def set_meta(self, key: str, value: str):
-        self.metadata[key] = value
-
-    def get_meta(self, key: str) -> str:
-        return self.metadata[key]
 
 
+# === Task Classes ===
 class Task:
     """
     Holds information about an AV-processing Task.
@@ -431,7 +503,12 @@ class Task:
     A Task is a collection of Input- and Output-Files and related options.
     While OutputFiles are bound to one task at a time, InputFiles can be reused across Tasks.
     """
+
+    output_factory = OutputFile
+
     def __init__(self, pp: "AdvancedAV"):
+        super().__init__()
+
         self.pp = pp
 
         self.inputs = []
@@ -451,11 +528,11 @@ class Task:
         :param file: Can be either the filename of an input file or an InputFile object.
                         The latter will be created if the former is passed.
         """
-        if not isinstance(file, InputFile):
+        if  isinstance(file, str):
             if file in self.inputs_by_name:
                 return self.inputs_by_name[file]
 
-            file = InputFile(self.pp, file)
+            file = self.pp.create_input(file)
 
         if file not in self.inputs:
             self.pp.to_debug("Adding input file #%i: %s", len(self.inputs), file.name)
@@ -483,7 +560,7 @@ class Task:
             if outfile.name == filename:
                 raise AdvancedAVError("Output File '%s' already added." % filename)
         else:
-            outfile = OutputFile(self, filename, container, options)
+            outfile = self.output_factory(self, filename, container, options)
             self.pp.to_debug("New output file #%i: %s", len(self.outputs), filename)
             self.outputs.append(outfile)
             return outfile
@@ -635,13 +712,16 @@ class SimpleTask(Task):
 
     container = _redir("output", "container")
     metadata = _redir("output", "metadata")
-    name = _redir("output", "name")
     options = _redir("output", "options")
+    name = _redir("output", "name")
 
     del _redir
 
 
+# === Interface Class ===
 class AdvancedAV(metaclass=ABCMeta):
+    input_factory = InputFile
+
     # ---- Output ----
     @abstractmethod
     def to_screen(self, text: str, *fmt):
@@ -692,7 +772,7 @@ class AdvancedAV(metaclass=ABCMeta):
         return SimpleTask(self, filename, container, options)
 
     # ---- Create InputFiles ----
-    def create_input(self, filename: str, options):
+    def create_input(self, filename: str, options=None):
         """
         Create a InputFile instance
         :param filename: str The filename
@@ -700,7 +780,7 @@ class AdvancedAV(metaclass=ABCMeta):
         :return: A InputFile instance
         NOTE that Task.add_input is usually the preferred way to create inputs
         """
-        return InputFile(self, filename, options)
+        return self.input_factory(pp=self, filename=filename, options=options)
 
 
 class SimpleAV(AdvancedAV):