diff --git a/resources/lib/updateFileDownload.py b/resources/lib/updateFileDownload.py index e7876a4..dda6b10 100644 --- a/resources/lib/updateFileDownload.py +++ b/resources/lib/updateFileDownload.py @@ -24,8 +24,6 @@ from codecs import open import resources.lib.mvutils as mvutils - -# from resources.lib.utils import * from resources.lib.exceptions import ExitRequested # -- Unpacker support --------------------------------------- @@ -79,47 +77,44 @@ def removeDownloads(self): mvutils.file_remove(self._filename) def downloadIncrementalUpdateFile(self): - # ext = self._getExtension() downloadUrl = FILMLISTE_URL + FILMLISTE_DIF + ext - self._compressedFilename = os.path.join(self.settings.getDatapath() , FILMLISTE_DIF + ext) - self._filename = os.path.join(self.settings.getDatapath() , FILMLISTE_DIF) - # + self._compressedFilename = os.path.join(self.settings.getDatapath(), FILMLISTE_DIF + ext) + self._filename = os.path.join(self.settings.getDatapath(), FILMLISTE_DIF) + check = self._download(downloadUrl, self._compressedFilename, self._filename) - # + return check def downloadFullUpdateFile(self): - # ext = self._getExtension() downloadUrl = FILMLISTE_URL + FILMLISTE_AKT + ext - self._compressedFilename = os.path.join(self.settings.getDatapath() , FILMLISTE_AKT + ext) - self._filename = os.path.join(self.settings.getDatapath() , FILMLISTE_AKT) - # + self._compressedFilename = os.path.join(self.settings.getDatapath(), FILMLISTE_AKT + ext) + self._filename = os.path.join(self.settings.getDatapath(), FILMLISTE_AKT) + check = self._download(downloadUrl, self._compressedFilename, self._filename) - # + if check: filesize = mvutils.file_size(self._filename) if filesize < 200000000: raise Exception('FullUpdate file size {} smaller than allowed (200MB)'.format(filesize)) - # + return check def downloadSqliteDb(self): ext = self._getExtension() downloadUrl = DATABASE_URL + DATABASE_DBF + ext - self._compressedFilename = os.path.join(self.settings.getDatapath() , 'tmp_' + DATABASE_DBF + ext) - self._filename = os.path.join(self.settings.getDatapath() , 'tmp_' + DATABASE_DBF) - self._Dbfilename = os.path.join(self.settings.getDatapath() , DATABASE_DBF) + self._compressedFilename = os.path.join(self.settings.getDatapath(), 'tmp_' + DATABASE_DBF + ext) + self._filename = os.path.join(self.settings.getDatapath(), 'tmp_' + DATABASE_DBF) + self._Dbfilename = os.path.join(self.settings.getDatapath(), DATABASE_DBF) - # check = self._download(downloadUrl, self._compressedFilename, self._filename) - # + if check: filesize = mvutils.file_size(self._filename) if filesize < 200000000: raise Exception('FullUpdate file size {} smaller than allowed (200MB)'.format(filesize)) - # + return check def updateSqliteDb(self): @@ -153,7 +148,7 @@ def _download(self, url, compressedFilename, targetFilename): # pylint: disable=broad-except try: self.logger.debug('Trying to download {} from {}...', - os.path.basename(compressedFilename), url) + os.path.basename(compressedFilename), url) self.notifier.update_download_progress(0, url) mvutils.url_retrieve( url, @@ -237,7 +232,7 @@ def _decompress_gz(self, sourcefile, destfile): mvutils.file_remove(destfile) retval = subprocess.call([gzip_binary, '-d', sourcefile]) self.logger.debug('Calling {} -d {} returned {}', - gzip_binary, sourcefile, retval) + gzip_binary, sourcefile, retval) return retval except Exception as err: self.logger.error( diff --git a/resources/lib/updateFileImport.py b/resources/lib/updateFileImport.py index 26d51a0..038f53f 100644 --- a/resources/lib/updateFileImport.py +++ b/resources/lib/updateFileImport.py @@ -57,7 +57,6 @@ def updateFull(self): self._update_end() def _importFile(self, targetFilename): - # if not mvutils.file_exists(targetFilename): self.logger.error('File {} does not exists!', targetFilename) return False @@ -65,19 +64,19 @@ def _importFile(self, targetFilename): fileSizeInByte = mvutils.file_size(targetFilename) records = int(fileSizeInByte / 600) self.logger.info('Starting import of approximately {} records from {}', records, targetFilename) - # + # pylint: disable=broad-except try: flsm = 0 flts = 0 - # + sender = "" thema = "" self.notifier.show_update_progress() - # + ufp = UpdateFileParser.UpdateFileParser(self.logger, 512000, targetFilename) ufp.init() - fileHeader = ufp.next('"X":'); + fileHeader = ufp.next('"X":') # META # {"Filmliste":["30.08.2020, 11:13","30.08.2020, 09:13","3","MSearch [Vers.: 3.1.139]","d93c9794acaf3e482d42c24e513f78a8"],"Filmliste":["Sender","Thema","Titel","Datum","Zeit","Dauer","Größe [MB]","Beschreibung","Url","Website","Url Untertitel","Url RTMP","Url Klein","Url RTMP Klein","Url HD","Url RTMP HD","DatumL","Url History","Geo","neu"] # this is the timestamp of this database update @@ -101,22 +100,21 @@ def _importFile(self, targetFilename): except Exception as err: # If the universe hates us... self.logger.debug('Could not determine date "{}" of filmliste: {}', value.strip(), err) - except ValueError as err: + except ValueError: pass - # - recordArray = []; - # + recordArray = [] + while (True): aPart = ufp.next('"X":') aPart = aPart.strip() if (len(aPart) == 0): - break; - # + break + aPart = '{"X":' + aPart aPart = aPart[0:-1] + '}' - # - #self.logger.debug('PARSE {}', aPart) + + # self.logger.debug('PARSE {}', aPart) jsonDoc = json.loads(aPart) jsonDoc = jsonDoc['X'] self._init_record() @@ -130,16 +128,16 @@ def _importFile(self, targetFilename): thema = jsonDoc[1][:128] else: jsonDoc[1] = thema - # + self.film['channel'] = sender self.film['show'] = thema self.film["title"] = jsonDoc[2][:128] - # + if len(jsonDoc[3]) == 10: self.film["aired"] = jsonDoc[3][6:] + '-' + jsonDoc[3][3:5] + '-' + jsonDoc[3][:2] if (len(jsonDoc[4]) == 8): self.film["aired"] = self.film["aired"] + " " + jsonDoc[4] - # + if len(jsonDoc[5]) > 0: self.film["duration"] = jsonDoc[5] if len(jsonDoc[7]) > 0: @@ -233,7 +231,9 @@ def _update_start(self): self._init_record() def _update_end(self): - self.logger.info('{} records processed in {} sec. Updated: {} Inserted: {} deleted: {}', self.count, int(time.time() - self.startTime), self.updateCount, self.insertCount, self.deletedCount) + self.logger.info('{} records processed in {} sec. Updated: {} Inserted: {} deleted: {}', + self.count, int(time.time() - self.startTime), + self.updateCount, self.insertCount, self.deletedCount) def _init_record(self): self.film["channel"] = "" diff --git a/resources/lib/updater.py b/resources/lib/updater.py index 60bb196..4191140 100644 --- a/resources/lib/updater.py +++ b/resources/lib/updater.py @@ -65,41 +65,41 @@ def doUpdate(self): tsnow = int(time.time()) currentDate = datetime.now() lastUpdateDatetime = datetime.fromtimestamp(databaseStatus['lastUpdate']) - # + outdated = ((databaseStatus['lastUpdate'] + self.settings.getDatabaseUpdateInvterval()) < tsnow) sameDay = (currentDate.day == lastUpdateDatetime.day and - currentDate.month == lastUpdateDatetime.month and - currentDate.year == lastUpdateDatetime.year) - # + currentDate.month == lastUpdateDatetime.month and + currentDate.year == lastUpdateDatetime.year) + self.logger.debug('Last Update {}', datetime.fromtimestamp(databaseStatus['lastUpdate'])) self.logger.debug('Last Full Update {}', datetime.fromtimestamp(databaseStatus['lastFullUpdate'])) self.logger.debug('version {}', databaseStatus['version']) self.logger.debug('status {}', databaseStatus['status']) self.logger.debug('update interval {}', self.settings.getDatabaseUpdateInvterval()) - # - updateConfigName = {0:"Disabled", 1:"Manual", 2:"On Start", 3:"Automatic", 4:"continuous"} + + updateConfigName = {0: "Disabled", 1: "Manual", 2: "On Start", 3: "Automatic", 4: "continuous"} self.logger.debug('Update Mode "{}"', updateConfigName.get(updateConfig)) - # + doSomething = 0 if (int(databaseStatus['version']) != 3 or databaseStatus['status'] == 'UNINIT'): self.logger.debug('Version update or not initialized') doSomething = -1 - # + if self.settings.getDatabaseType() == 0: StoreSQLiteSetup(self.database).setupDatabase() else: StoreMySQLSetup(self.database).setupDatabase() - # + self.database.set_status(pStatus='IDLE', pLastupdate=0, pLastFullUpdate=0, pFilmupdate=0, pVersion='3') databaseStatus = self.database.getDatabaseStatus() - # + elif updateConfig == 1 and self.settings.is_update_triggered(): self.logger.debug('Manual update') doSomething = 1 elif updateConfig == 2 and self.settings.is_update_triggered(): self.logger.debug('On Start update - was triggered manual') doSomething = 1 - elif updateConfig == 2 and not(sameDay) and self.settings.is_user_alive(): + elif updateConfig == 2 and not sameDay and self.settings.is_user_alive(): self.logger.debug('On Start update and no update today') doSomething = 1 elif updateConfig == 3 and self.settings.is_user_alive() and outdated: @@ -115,16 +115,14 @@ def doUpdate(self): if (doSomething == 0): self.logger.debug('nothing to do') return - # + lastFullUpdate = datetime.fromtimestamp(databaseStatus['lastFullUpdate']) - # ufd = UpdateFileDownload() - # + if doSomething == -1 or (not (currentDate.day == lastFullUpdate.day and - currentDate.month == lastFullUpdate.month and - currentDate.year == lastFullUpdate.year) and - currentDate.hour > 5 - ): + currentDate.month == lastFullUpdate.month and + currentDate.year == lastFullUpdate.year) and + currentDate.hour > 5): if self.settings.getDatabaseType() == 0 and self.settings.getDatabaseUpdateNative(): # replace the sqlite DB by downloaded version self.logger.debug('sqlite update') @@ -138,24 +136,23 @@ def doUpdate(self): self.database.set_status('IDLE', pLastupdate=int(time.time()), pLastFullUpdate=int(time.time())) else: self.database.set_status('UNINIT') - # + self.settings.set_update_triggered('false') else: # download full filmlist and do a full update self.logger.debug('full update') - if (not(mvutils.file_exists(os.path.join(self.settings.getDatapath() , 'Filmliste-akt')))): + if not mvutils.file_exists(os.path.join(self.settings.getDatapath(), 'Filmliste-akt')): ufd.downloadFullUpdateFile() downloadFullUpdate = True else: - ufd._filename = os.path.join(self.settings.getDatapath() , 'Filmliste-akt') + ufd._filename = os.path.join(self.settings.getDatapath(), 'Filmliste-akt') self.logger.debug('use existing full update file') downloadFullUpdate = False UpdateFileImport(ufd.getTargetFilename(), self.database).updateFull() - if (downloadFullUpdate): + if downloadFullUpdate: ufd.removeDownloads() - # + self.database.set_status('IDLE', pLastupdate=int(time.time()), pLastFullUpdate=int(time.time())) - # self.settings.set_update_triggered('false') else: # download incremental filmlist and do the update @@ -165,4 +162,3 @@ def doUpdate(self): ufd.removeDownloads() self.database.set_status('IDLE', pLastupdate=int(time.time())) self.settings.set_update_triggered('false') - diff --git a/resources/lib/vtt2srt.py b/resources/lib/vtt2srt.py index 2bfa3d1..d613f15 100644 --- a/resources/lib/vtt2srt.py +++ b/resources/lib/vtt2srt.py @@ -2,15 +2,16 @@ """ VTT to SRT conversion module -Thanks to (c) Jansen A. Simanullang for his function taken from vtt-to-srt project +Thanks to (c) Jansen A. Simanullang for his function taken from vtt-to-srt project See https://github.com/jansenicus/vtt-to-srt.py SPDX-License-Identifier: MIT """ import re + class vtt2srt(object): - + def convertContent(self, fileContents): replacement = re.sub(r'(\d\d:\d\d:\d\d).(\d\d\d) --> (\d\d:\d\d:\d\d).(\d\d\d)(?:[ \-\w]+:[\w\%\d:]+)*\n', r'\1,\2 --> \3,\4\n', fileContents) @@ -19,12 +20,12 @@ def convertContent(self, fileContents): replacement = re.sub(r'WEBVTT\n', '', replacement) replacement = re.sub(r'Kind:[ \-\w]+\n', '', replacement) replacement = re.sub(r'Language:[ \-\w]+\n', '', replacement) - #replacement = re.sub(r'^\d+\n', '', replacement) - #replacement = re.sub(r'\n\d+\n', '\n', replacement) + # replacement = re.sub(r'^\d+\n', '', replacement) + # replacement = re.sub(r'\n\d+\n', '\n', replacement) replacement = re.sub(r'', '', replacement) replacement = re.sub(r'', '', replacement) replacement = re.sub(r'<\d\d:\d\d:\d\d.\d\d\d>', '', replacement) replacement = re.sub(r'::[\-\w]+\([\-.\w\d]+\)[ ]*{[.,:;\(\) \-\w\d]+\n }\n', '', replacement) replacement = re.sub(r'Style:\n##\n', '', replacement) - + return replacement