Mercurial > hg-public > dreambox_tools / changeset
changeset
code cleanup
Tue, 28 Nov 2017 22:27:01 +0100
- author
- mdd
- date
- Tue, 28 Nov 2017 22:27:01 +0100
- changeset 17
- 842120f00078
- parent 16
- ace8005f02cf
- child 18
- 91cf8d348c74
code cleanup
| eit.py | file | annotate | diff | comparison | revisions | |
| ts2mkv.py | file | annotate | diff | comparison | revisions |
--- a/eit.py Tue Nov 28 21:04:03 2017 +0100 +++ b/eit.py Tue Nov 28 22:27:01 2017 +0100 @@ -146,11 +146,13 @@ self.eit_file = None self.eit = {} - self.iso = None + #self.iso = None self.load(path) def load(self, path): + self.eit = {} + self.eit_file = None if path: self.eit_file = path self._read_file() @@ -188,8 +190,22 @@ def get_date(self): return todate(self.get_startdate(), self.get_starttime()) - def dumpEit(self): - print self.eit + def dump(self): + """Module docstring. + Read Eit File and show the information. + """ + if len(self.eit) == 0: + return None + out = "Movie name: %s" % self.get_name() + out += "\nGenre: %s" % self.get_genre() + out += "\nComponents: %s" % self.get_components() + out += "\nStartDate: %s" % self.get_date() + out += "\nDescription: %s" % self.get_description() + out += "\nDuration: %02i:%02i:%02i" % self.get_duration() + out += " (%s minutes)" % (self.get_duration_seconds() / 60) + + print out + return out ############################################################################## ## File IO Functions @@ -355,23 +371,6 @@ self.eit = {} -def readeit(eitfile): - """Module docstring. - Read Eit File and show the information. - """ - eitlist = eitinfo(eitfile) - if len(eitlist.eit) == 0: - return None - out = "Movie name: %s" % eitlist.get_name() - out += "\nGenre: %s" % eitlist.get_genre() - out += "\nComponents: %s" % eitlist.get_components() - out += "\nStartDate: %s" % eitlist.get_date() - out += "\nDescription: %s" % eitlist.get_description() - out += "\nDuration: %02i:%02i:%02i" % eitlist.get_duration() - out += " (%s minutes)" % (eitlist.get_duration_seconds() / 60) - - print out - return out def main(): # parse command line options @@ -387,8 +386,10 @@ print __doc__ sys.exit(0) # process arguments + info = eitinfo() for arg in args: - readeit(arg) # process() is defined elsewhere + info.load(arg) + info.dump() if __name__ == "__main__": main()
--- a/ts2mkv.py Tue Nov 28 21:04:03 2017 +0100 +++ b/ts2mkv.py Tue Nov 28 22:27:01 2017 +0100 @@ -4,19 +4,25 @@ 2017 by mdd Toolkit / executable to automagically convert DVB recordings to h264 mkv. -Automatic audio stream selection (deu/eng) +Automatic audio stream selection + deu: ac3, otherwise fallback to first german stream + eng: ac3, no fallback Automatic crop detection to remove cinematic bars +percentage + ETA for ffmpeg conversion subprocess """ +#pylint: disable=line-too-long +#pylint: disable=invalid-name + import subprocess import pexpect -from eit import readeit, eitinfo +from eit import eitinfo import os, shlex, sys, time def filter_lines(data, search): """ input: data = \n separated string - output: tuple containing all lines where search is found + output: all lines where search is found """ ret = [] for line in data.split("\n"): @@ -41,11 +47,13 @@ rc = process.poll() return rc -def run_ffmpeg_watch(command, frames_total = 0): +def run_ffmpeg_watch(command, frames_total=0): """ run command as blocking subprocess, returns exit code if total_frames > 0 parse ffmpeg status line and insert ETA at line start before output """ + #pylint: disable=maybe-no-member + thread = pexpect.spawn(command) cpl = thread.compile_pattern_list([ pexpect.EOF, @@ -72,56 +80,66 @@ sys.stdout.write("\rFrame %i of %i, %.1f%% done, ETA %.0f minutes, " % ( frame_number, frames_total, percent, eta )) - except: + except ValueError: sys.stdout.write(thread.match.group(0)) sys.stdout.flush() thread.close - elif i == 2: + #elif i == 2: # normal newline line, just ignore them... - pass + # pass elif i == 3: unknown_line = thread.match.group(0) sys.stdout.write(unknown_line) sys.stdout.flush() - pass def ffmpeg_filename(filename): """ Escape filename path contents for ffmpeg shell command """ - #fn = "\\'".join(p for p in filename.split("'")) - fn = filename.replace("'", "\\'") - fn = fn.replace(" ", "\\ ") + fn = filename.replace("'", r"\'") + fn = fn.replace(" ", r"\ ") return fn class ts2mkv(object): """ Main worker class, contains all the magic & ffmpeg voodoo """ - def __init__(self, crf=19, tune='film', scaleto_720p=True, rename=False): - self.msg_prepare = "" - self.msg_eit = "" - self.msg_ffmpeg = "" + def __init__(self, crf=19, tune='film'): self.command = None self.filename = None self.outfilebase = None - self.fps = 0 - self.frames_total = 0 - self.overwrite = False - - self.scaleto_720p = scaleto_720p - self.rename = rename + self.info = {} + self.__reset() - self.video_options = [ - "-c:v libx264", - "-preset faster", # slow - "-tune %s" % tune, # film / animation - "-crf %i" % crf, # 21, better 19 - ] - self.audio_options = [ - "-c:a copy", - ] + self.config = { + "overwrite": False, + "scaledown": True, + "rename": True, + "video": [ + "-c:v libx264", + "-preset faster", # slow + "-tune %s" % tune, # film / animation + "-crf %i" % crf, # 21, better 19 + ], + "audio": [ + "-c:a copy", + ] + } + def __reset(self): + """ + Reset internal stuff before loading new task + """ + self.info = { + "msg_prepare": "", + "msg_eit": "", + "msg_ffmpeg": "", + "fps": 0, + "frames_total": 0 + } + self.command = None + self.filename = None + self.outfilebase = None def get_stream_index(self, data): """ @@ -132,10 +150,10 @@ if idx == -1: return "" idx += 8 - self.msg_prepare += "Selecting: %s\n" % data + self.info["msg_prepare"] += "Selecting: %s\n" % data return data[idx:idx+3] - def get_movie_description(self): + def __get_movie_description(self): """ looks for eit file with same basename of current filename parse the eit file for txt infofile and optional build new @@ -147,10 +165,10 @@ return # read the EIT file filename = os.path.splitext(self.filename)[0] + ".eit" - self.msg_eit = readeit(filename) - if not self.rename or not self.msg_eit: + info = eitinfo(filename) + self.info["msg_eit"] = info.dump() + if not self.config["rename"] or not self.info["msg_eit"]: return - info = eitinfo(filename) name = info.eit.get("name") if name == "": # cancel rename, no movie title found! @@ -179,7 +197,7 @@ parse the ffmpeg analyze output cropdetect lines returns None or valid crop string for ffmpeg video filter """ - lines = filter_lines(self.msg_ffmpeg, "[Parsed_cropdetect").split("\n") + lines = filter_lines(self.info["msg_ffmpeg"], "[Parsed_cropdetect").split("\n") option = None failcount = 0 for line in lines: @@ -192,9 +210,9 @@ failcount += 1 if failcount > 12: print "!!! Crop detect is inconsistent" - self.msg_prepare += "WARNING: cropdetect >50% inconsistent over scan time, disabling autocrop\n" + self.info["msg_prepare"] += "WARNING: cropdetect >50% inconsistent over scan time, disabling autocrop\n" return None - self.msg_prepare += "Crop detected: %s\n" % option + self.info["msg_prepare"] += "Crop detected: %s\n" % option return option def __get_audiomap(self, info): @@ -229,6 +247,54 @@ audiomap.append(aidx) return audiomap + def __parse_info(self): + """ + get total duration and fps from input stream + output: sets local variables + # Duration: 01:39:59.88, start: 93674.825111, bitrate: 9365 kb/s + # Stream #0:1[0x1ff]: Video: h264 (High) ([27][0][0][0] / 0x001B), yuv420p(tv, bt709), 1920x1080 [SAR 1:1 DAR 16:9], 25 fps, 50 tbr, 90k tbn, 50 tbc + """ + tmp = filter_lines(self.info["msg_ffmpeg"], "Duration:").strip()[10:] + tmp = tmp[0:tmp.find(",")].strip() + print "Input duration: %s" % tmp + try: + self.info["frames_total"] = int(tmp[0:2]) * 3600 + \ + int(tmp[3:5]) * 60 + int(tmp[6:8]) + except ValueError: + self.info["frames_total"] = 0 + + tmp = filter_lines(self.info["msg_ffmpeg"], "Stream #0:") + tmp = filter_lines(tmp, "Video:").split(",") + for fps in tmp: + if fps.strip().endswith('fps'): + try: + self.info["fps"] = float(fps.strip().split(' ')[0]) + except ValueError: + self.info["fps"] = 0 + break + self.info["frames_total"] = round(self.info["frames_total"] * self.info["fps"], 0) + print "Input framerate: %f fps" % self.info["fps"] + print "Total frames of input file: %i" % (self.info["frames_total"]) + + + def __get_ffmpeg_input_info(self, filename): + """ + Run ffmpeg for cropdetect and general input information + """ + cmd = [ + "ffmpeg", "-hide_banner", + "-ss 00:05:00", "-t 2", # search to 5 minutes, analyze 2 seconds + "-i %s" % filename, + "-vf \"cropdetect=24:2:0\"", # detect black bar crop on top and bottom + "-f null", "-" # no output file + ] + p = subprocess.Popen(shlex.split(" ".join(cmd)), \ + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + with self.info["msg_ffmpeg"] as msg: + msg = out + "\n" + err + msg = msg[msg.find("Input #0"):] + def get_ffmpeg_command(self): """ Too complex to describe, this does all the magic @@ -237,33 +303,23 @@ if not self.filename: return None + fn = { + "in": ffmpeg_filename(self.filename), + "out": self.outfilebase + ".mkv" + } - commands = [] - fn = ffmpeg_filename(self.filename) - outfn = self.outfilebase + ".mkv" # double-check: pull the kill switch and exit if outfile exists already! # we do not want to overwrite files in accident (caused by automatic file naming) - if not self.overwrite and len(glob.glob(outfn)) > 0: - print "Output file exists: %s" % outfn + if not self.config["overwrite"] and len(glob.glob(fn["out"])) > 0: + print "Output file exists: %s" % fn["out"] print "NOT overwriting it!" return None - outfn = ffmpeg_filename(outfn) - cmd = [ - "ffmpeg", "-hide_banner", - "-ss 00:05:00", "-t 2", # search to 5 minutes, analyze 2 seconds - "-i %s" % fn, - "-vf \"cropdetect=24:2:0\"", # detect black bar crop on top and bottom - "-f null", "-" # no output file - ] - p = subprocess.Popen(shlex.split(" ".join(cmd)), \ - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - self.msg_ffmpeg = out + "\n" + err - self.msg_ffmpeg = self.msg_ffmpeg[self.msg_ffmpeg.find("Input #0"):] + # load input file to get informations about + self.__get_ffmpeg_input_info(fn["in"]) # find "Stream #0:" lines - info = filter_lines(self.msg_ffmpeg, "Stream #0:") + info = filter_lines(self.info["msg_ffmpeg"], "Stream #0:") v = self.get_stream_index( filter_lines(info, "Video:")) @@ -271,30 +327,7 @@ print "No video stream found" return None - # get total duration and fps from input stream - # Input #0, mpegts, from '/srv/storage0/DREAMBOX/Action/Transporter/20101201 0630 - Sky Action HD - Transporter 3.ts': - # Duration: 01:39:59.88, start: 93674.825111, bitrate: 9365 kb/s - # Stream #0:1[0x1ff]: Video: h264 (High) ([27][0][0][0] / 0x001B), yuv420p(tv, bt709), 1920x1080 [SAR 1:1 DAR 16:9], 25 fps, 50 tbr, 90k tbn, 50 tbc - self.frames_total = filter_lines(self.msg_ffmpeg, "Duration:").strip()[10:] - self.frames_total = self.frames_total[0:self.frames_total.find(",")].strip() - print "Input duration: %s" % self.frames_total - try: - self.frames_total = int(self.frames_total[0:2]) * 3600 + \ - int(self.frames_total[3:5]) * 60 + int(self.frames_total[6:8]) - except ValueError: - self.frames_total = 0 - - tmp = filter_lines(info, "Video:").split(",") - for fps in tmp: - if fps.strip().endswith('fps'): - try: - self.fps = float(fps.strip().split(' ')[0]) - except ValueError: - self.fps = 0 - break - self.frames_total = round(self.frames_total * self.fps, 0) - print "Input framerate: %f fps" % self.fps - print "Total frames of input file: %i" % (self.frames_total) + self.__parse_info() # copy ALL subtitle streams if present! # Stream #0:0[0x20](deu): Subtitle: dvb_teletext ([6][0][0][0] / 0x0006), 492x250 @@ -313,83 +346,77 @@ # Old dreambox images did a file split: .ts .ts.001 .ts.002 etc. # Find all these files and join them! - inputs = [fn] - if os.path.splitext(fn)[1].lower() == '.ts': - for fpart in glob.glob(self.filename + '.' + ('[0-9]' * 3)): - fn = "\\'".join(p for p in fpart.split("'")) - fn = fn.replace(" ", "\\ ") - inputs.append(fn) + inputs = [fn["in"]] + if os.path.splitext(self.filename)[1].lower() == '.ts': + for tmp in glob.glob(self.filename + '.' + ('[0-9]' * 3)): + inputs.append(ffmpeg_filename(tmp)) if len(inputs) > 1: # use ffmpeg input concat function - # attention, ffmpeg doesnt like escape sequences - fn = "\"concat:" + \ + # attention, ffmpeg concat protocol doesnt like escape sequences + fn["in"] = "\"concat:" + \ "|".join(inputs)\ - .replace('\ ', ' ')\ - .replace("\'", "'")\ + .replace(r"\ ", " ")\ + .replace(r"\'", "'")\ + "\"" # no ETA calculation possible since we have only the length of first file # TODO: we COULD estimate by multiplying with factor generated by input file sizes - print "NO ETA POSSIBLE" - self.frames_total = 0 + print "No ETA info possible yet on split input" + self.info["frames_total"] = 0 idx = 0 for tmp in inputs: - self.msg_prepare += "Input file #%i: %s\n" % ( + self.info["msg_prepare"] += "Input file #%i: %s\n" % ( idx, os.path.basename(tmp)) idx += 1 cmd = [ "ffmpeg", "-hide_banner", - "-i %s" % fn, + "-i %s" % fn["in"], ] - if self.overwrite: + if self.config["overwrite"]: cmd.append("-y") for tmp in submap: - self.msg_prepare += "Subtitle Stream selected: Stream #%s\n" % tmp + self.info["msg_prepare"] += "Subtitle Stream selected: Stream #%s\n" % tmp cmd.append("-map %s" % tmp) cmd.append("-map %s" % v) - self.msg_prepare += "Video Stream selected: Stream #%s\n" % v + self.info["msg_prepare"] += "Video Stream selected: Stream #%s\n" % v flt = [] crop = self.get_crop_option() if crop: flt.append(crop) - if self.scaleto_720p: + if self.config["scaledown"]: # -2 ensures division by two for codec flt.append("scale='min(1280,iw)':-2'") - self.msg_prepare += "Scaling output stream to 720p if width >1280\n" + self.info["msg_prepare"] += "Scaling output stream to 720p if width >1280\n" if len(flt) > 0: # append video filters cmd.append('-filter:v "%s"' % ",".join(flt)) + for tmp in audiomap: - self.msg_prepare += "Audio Stream selected: Stream #%s\n" % tmp + self.info["msg_prepare"] += "Audio Stream selected: Stream #%s\n" % tmp cmd.append("-map %s" % tmp) if len(submap) > 0: cmd.append("-c:s dvdsub") - cmd.extend(self.video_options) - cmd.extend(self.audio_options) - cmd.append(outfn) + cmd.extend(self.config["video"]) + cmd.extend(self.config["audio"]) + cmd.append(ffmpeg_filename(fn["out"])) - commands.append(" ".join(cmd)) - return commands + return [" ".join(cmd)] def load(self, filename): """ First step: setup, analyze & prepare for conversion """ - self.msg_prepare = "" - self.msg_eit = "" - self.msg_ffmpeg = "" - self.fps = 0 - self.frames_total = 0 + self.__reset() self.filename = filename self.outfilebase = os.path.splitext(filename)[0] - self.get_movie_description() + self.__get_movie_description() self.command = self.get_ffmpeg_command() def convert(self): @@ -401,17 +428,17 @@ if not self.command: return None fd = open(self.outfilebase + ".txt", "wb") - fd.write(self.msg_eit) + fd.write(self.info["msg_eit"]) fd.write("\n\n# ---DEBUG---\n\n") - fd.write(self.msg_prepare) - fd.write(self.msg_ffmpeg) + fd.write(self.info["msg_prepare"]) + fd.write(self.info["msg_ffmpeg"]) fd.close() - #print self.msg_ffmpeg + #print self.info["msg_ffmpeg"] for cmd in self.command: print "Executing ffmpeg:\n%s\n" % cmd #return run_command(cmd, self.total_frames) - return run_ffmpeg_watch(cmd, frames_total=self.frames_total) + return run_ffmpeg_watch(cmd, frames_total=self.info["frames_total"]) @@ -434,9 +461,11 @@ help='force overwrite of existing file') args = parser.parse_args() - processor = ts2mkv(crf=args.crf, tune=args.tune, scaleto_720p=(not args.ns), \ - rename=args.rename) - processor.overwrite = args.f + processor = ts2mkv(crf=args.crf, tune=args.tune) + with processor.config as c: + c["scaledown"] = not args.ns + c["rename"] = args.rename + c["overwrite"] = args.f for srcstr in args.input: src = glob.glob(srcstr)
