dreambox_tools: comparison dupecheck.py

-:14c966c10648
+:a1ad6f4728be
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 """
 Toolkit / executable to scan for duplicate filenames in movie database
-2017 by mdd
+2017-2019 by mdd
 """
 #pylint: disable=line-too-long
 #pylint: disable=invalid-name
 from __future__ import print_function
 import os, sys, re
+import time
 RE_PARENTHESES = re.compile("[\(\[].*?[\)\]]")
 def similarity(a, b):
 if DIFFLIB:
 self.basedir = ""
 self.filelist = []
 self.duplicates = {}
 self.ratio = 0.85
 self.ignore_fileprefix = []
+self.ssh = None
+self.ssh_data = None
 def reset(self):
 self.filelist = []
 self.duplicates = {}
+def __scandir_files(self, root, files, extra=[]):
+for filename in files:
+ext = os.path.splitext(filename)[1].lower()
+if ext == ".ts":
+#file_path = os.path.join(root, filename)
+title = filename.split(" - ")
+if len(title) == 1:
+title = title[0]
+else:
+title = " - ".join(title[2:])
+title = title[:-3].lower()
+# remove parentheses with contents in title
+title = RE_PARENTHESES.sub("", title)
+self.filelist.append([title, filename, root, ext])
+elif ext in ['.mkv', '.avi', '.mpg', '.mpeg', '.mp4']:
+title = filename[:-4].lower()
+title = RE_PARENTHESES.sub("", title)
+self.filelist.append([title, filename, root, ext])
+elif ext in extra:
+title = filename[:-4].lower()
+title = RE_PARENTHESES.sub("", title)
+self.filelist.append([title, filename, root, ext])
 def scandir(self, basedir, extra=[]):
 """
 Scan a base directory for movie files and add them to
 the list for analyze
 """
 self.basedir = basedir
 print("Scanning directory: %s" % basedir)
 for root, subdirs, files in os.walk(basedir):
-for filename in files:
+self.__scandir_files(root, files, extra)
-ext = os.path.splitext(filename)[1].lower()
+# print(repr(self.filelist))
-if ext == ".ts":
+# sys.exit()
-#file_path = os.path.join(root, filename)
-title = filename.split(" - ")
+def scandir_remote(self, extra=[]):
-if len(title) == 1:
+"""
-title = title[0]
+connect to remote ssh servers and get file lists for duplicate check
-else:
+"""
-title = " - ".join(title[2:])
+print("getting filelist from remote hosts...")
-title = title[:-3].lower()
+try:
+from config import REMOTE_HOSTS
-# remove parentheses with contents in title
+except ImportError:
-title = RE_PARENTHESES.sub("", title)
+print("Please configure REMOTE_HOSTS in config.py!")
+sys.exit(1)
-self.filelist.append([title, filename, root, ext])
+try:
-elif ext in ['.mkv', '.avi', '.mpg', '.mpeg', '.mp4']:
+import paramiko
-title = filename[:-4].lower()
+self.ssh = paramiko.SSHClient()
-title = RE_PARENTHESES.sub("", title)
+#self.ssh.set_missing_host_key_policy(paramiko.WarningPolicy())
-self.filelist.append([title, filename, root, ext])
+self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-elif ext in extra:
+#self.ssh_key = paramiko.RSAKey.from_private_key_file(SSH_PRIVATE_KEY_FILE)
-title = filename[:-4].lower()
+except ImportError:
-title = RE_PARENTHESES.sub("", title)
+print("Please install Paramiko!")
-self.filelist.append([title, filename, root, ext])
+sys.exit(1)
+for host in REMOTE_HOSTS:
+self.ssh_data = host
+cleanlist = []
+lst = self.__ssh_exec('cd %s; ls -1 *.ts' % self.ssh_data['basedir'])[0]
+for item in lst:
+cleanlist.append(item.strip().encode('ascii','ignore'))
+self.__scandir_files("%s: %s" % (
+self.ssh_data['host'], self.ssh_data['basedir']), cleanlist)
+# self.__scandir_files(self.ssh_data['basedir'], cleanlist)
+self.__ssh_disconnect()
+def __ssh_exec(self, command):
+"""
+establish ssh connection and execute command
+the connection remains open for following commands until ssh_disconnect is called
+"""
+if self.ssh is None:
+return None
+try:
+transport = self.ssh.get_transport()
+if not transport or not transport.is_active():
+print("SSH: connecting to %s" % self.ssh_data['host'])
+self.ssh.connect(self.ssh_data['host'], self.ssh_data['port'], self.ssh_data['user'], self.ssh_data['pass'], self.ssh_data['key'])
+# Send the command (non-blocking)
+stdin, stdout, stderr = self.ssh.exec_command(command)
+# Wait for the command to terminate
+while not stdout.channel.exit_status_ready() and not stdout.channel.recv_ready():
+time.sleep(1)
+stdoutstring = stdout.readlines()
+stderrstring = stderr.readlines()
+return stdoutstring, stderrstring
+finally:
+pass
+def __ssh_disconnect(self):
+"""
+check if ssh is connected and disconnect
+"""
+if self.ssh is not None:
+# Close client connection.
+transport = self.ssh.get_transport()
+if not transport or not transport.is_active():
+print("SSH: disconnecting")
+self.ssh.close()
 def fixnames(self):
 """
 Search for defect filenames and remove illegal characters
 """
 help='filename duplicate threshold 0.1 < ratio 1.0 (default 0.85)')
 parser.add_argument('--difflib', action='store_true', default=False, \
 help='force the use of difflib instead Levenshtein')
 parser.add_argument('--stats', action='store_true', default=False, \
 help='generate stats summary instead of check for duplicates')
+parser.add_argument('--remote', action='store_true', default=False, \
+help='Connect to ssh remotes, eg. dupecheck for dreambox local storage')
 parser.add_argument('--fixnames', action='store_true', default=False, \
 help='scan for mkv and txt, fix broken filenames for windows')
 parser.add_argument('basedir', metavar='basedir', nargs='+', \
 help='one or more base directories')
 print ("Checking %i file names..." % len(dupe.filelist))
 dupe.fixnames()
 dupe.filelist = []
 sys.exit(0)
+if args.remote:
+dupe.scandir_remote()
 for srcstr in args.basedir:
 dupe.scandir(srcstr)
 if args.stats or args.fixnames:
 dupe.statistics()

Mercurial > hg-public > dreambox_tools / file comparison

comparison: dupecheck.py

dupecheck.py