Starting the refactor of this project

Chris Pomeroy

2023-11-30 1bc382c0c65a3721982861d16abdeeecd651320e

Starting the refactor of this project

 aaxConvert.py

@@ -7,16 +7,20 @@
import re
import requests
import sys
from querysubsonic import findalbumbyname
from queryAudiobookServer import findalbumbyname

# arguments
# activation_key, file name, codec(default to mp3)

parser = argparse.ArgumentParser()
parser.add_argument("-s", "--single", help="Use this option to create a single file. This is false by default", action="store_true")
parser.add_argument("-d", "--dpath", help="Use this to set the destination path. Otherwise I will use the current directory")
parser.add_argument("-v", "--verbose", help="Send output to stdout", action="store_true")
parser.add_argument("filename", help="Filename to convert, or directory to look in")
parser.add_argument("-s", "--single", help="Use this option to create a single\
                     file. This is false by default", action="store_true")
parser.add_argument("-d", "--dpath", help="Use this to set the destination\
                    path. Otherwise I will use the current directory")
parser.add_argument("-v", "--verbose", help="Send output to stdout", 
                    action="store_true")
parser.add_argument("filename", help="Filename to convert, or directory to\
                     look in")

args = parser.parse_args()

@@ -43,8 +47,10 @@

def getmetadata(aaxfile):
    # Returns the metadata from an aax file
    ret = subprocess.run(["ffprobe", "-v", "info", "-hide_banner", "-show_format", "-show_chapters",
                          "-print_format", "json", os.path.abspath(aaxfile)], capture_output=True)
    ret = subprocess.run(["ffprobe", "-v", "info", "-hide_banner",
                          "-show_format", "-show_chapters",
                          "-print_format", "json", 
                          os.path.abspath(aaxfile)], capture_output=True)

    mdata = json.loads(ret.stdout)
    aret = ret.stderr.decode().split('\n')[0]
@@ -79,13 +85,18 @@

def reencode(aaxfile, outpath):
    # decrypt and reencode to mp3
    command = ("ffmpeg -loglevel error {} -activation_bytes {} -i {} -vn -codec:a libmp3lame -ab {}k -map_metadata -1 "
               "-metadata \"title={}\" -metadata 'artist={}' -metadata 'album_artist={}' -metadata \"album={}\" -metadata 'date={}' "
               "-metadata track=1/1 -metadata 'genre={}' -metadata 'copyright={}' \"{}\" ").format(stats, act_byte, aaxfile, getmetabitrate(),
                                                                                                   getmetadatatags('title'), getmetadatatags('artist'),
                                                                                                   getmetadatatags('album_artist'), getmetadatatags('album'),
                                                                                                   getmetadatatags('date'), getmetadatatags('genre'),
                                                                                                   getmetacopyright(), outpath)
    command = (
        "ffmpeg -loglevel error {} -activation_bytes {} -i"
        " {} -vn -codec:a libmp3lame -ab {}k -map_metadata -1"
        "-metadata \"title={}\" -metadata 'artist={}' -metadata"
        " 'album_artist={}' -metadata \"album={}\" -metadata 'date={}' "
        "-metadata track=1/1 -metadata 'genre={}' -metadata "
        "'copyright={}' \"{}\" ").format(
            stats, act_byte, aaxfile, getmetabitrate(),
            getmetadatatags('title'), getmetadatatags('artist'),
            getmetadatatags('album_artist'), getmetadatatags('album'),
            getmetadatatags('date'), getmetadatatags('genre'),
            getmetacopyright(), outpath)
    if args.verbose:
        print(command)
        process = subprocess.run(command, shell=True, capture_output=True)
@@ -118,7 +129,9 @@
def movetochapters(path, outpath, chapter, title, start, end):
    # Creating individual chapters

    outfile = "{}/Ch-{}_{}.mp3".format(outpath, chapter, title.replace(' ', '_'))
    outfile = "{}/Ch-{}_{}.mp3".format(
        outpath, chapter, title.replace(' ', '_')
    )
    command = "ffmpeg -loglevel error {} -i \"{}\" -ss {} -to {} -codec:a copy -metadata 'track={}' \"{}\"".format(stats, path,
                                                                                                                   start, end,
                                                                                                                   chapter, outfile)
@@ -151,42 +164,46 @@
def getcorrectkey():
    # request the key for the checksum
    try:
        r = requests.post('http://faas.darkurthe.net/function/checkkey', metadata['checksum'], verify=False, timeout=None)
        r = requests.post('http://faas.darkurthe.net/function/checkkey',
                          metadata['checksum'], verify=False, timeout=None)
        return r.text.strip()
    except requests.exceptions.HTTPError as err:
        raise err
        return None

def findalbumbyname_stub(album):
    return False


for rfile in glob.glob(args.filename):
    if rfile.find("aax") != -1 and os.path.isfile(rfile):
        metadata = getmetadata(rfile)
        album = getmetadatatags('album')
        # See if we got it already
        if not findalbumbyname(album):
            artist = normalize_data(getmetadatatags('artist'))
            title = normalize_data(getmetadatatags('title'))
            act_byte = getcorrectkey()
            if act_byte is None:
                sys.exit("Can't continue with this file {rfile}")
            else:
                ddir = "%s/%s/%s" % (path, artist, title)
                single_file_path = "/processing/%s.mp3" % (title)
                if not os.path.exists(ddir):
                    os.makedirs(ddir)
                print(ddir)
                reencode(rfile, single_file_path)
                if mode == 'chapter':
                    chapter = 0
                    numchapters = getchaptercount()
                    while (numchapters > 0):
                        cstart = getchaptermetadata(chapter, 'start_time')
                        cend = getchaptermetadata(chapter, 'end_time')
                        chapter += 1
                        numchapters -= 1
                        schap = str(chapter).zfill(2)
                        movetochapters(single_file_path, ddir, schap, title, cstart, cend)
                    os.remove(single_file_path)
                getcoverart(rfile, ddir)
if args.filename.find("aax"):
    rfile = args.filename
    metadata = getmetadata(rfile)
    album = getmetadatatags('album')
    # See if we got it already
    if not findalbumbyname(album):
        artist = normalize_data(getmetadatatags('artist'))
        title = normalize_data(getmetadatatags('title'))
        act_byte = getcorrectkey()
        if act_byte is None or act_byte == '':
            sys.exit(f"Can't continue with this file {rfile}")
        else:
            print('We have that book already')
            ddir = "%s/%s/%s" % (path, artist, title)
            single_file_path = "/processing/%s.mp3" % (title)
            if not os.path.exists(ddir):
                os.makedirs(ddir)
            print(ddir)
            reencode(rfile, single_file_path)
            if mode == 'chapter':
                chapter = 0
                numchapters = getchaptercount()
                while (numchapters > 0):
                    cstart = getchaptermetadata(chapter, 'start_time')
                    cend = getchaptermetadata(chapter, 'end_time')
                    chapter += 1
                    numchapters -= 1
                    schap = str(chapter).zfill(2)
                    movetochapters(single_file_path, ddir, schap, title,
                                   cstart, cend)
                os.remove(single_file_path)
            getcoverart(rfile, ddir)
    else:
        print('We have that book already')

 filekeeper.py

New file
@@ -0,0 +1,170 @@
#!/usr/local/bin/python -u

import os
import sys
import subprocess
import shutil
import time
import hashlib

# path = "/home/audiobooks/Incoming"
path = str(os.environ['AAXPATH'])
archivepath = str(os.environ['ARCHIVEPATH'])
dpath = str(os.environ['DESTPATH'])
errorinprocessing = False


def getMD5(fpathaudiofile):
    with open(fpathaudiofile, "rb") as f:
        file_hash = hashlib.md5()
        while chunk := f.read(8192):
            file_hash.update(chunk)
    return file_hash.digest()


def changebadfilename(fpathaudiofile):
    # This is used in the case of a zero length file
    os.rename(fpathaudiofile, str(fpathaudiofile) + ".bad")


def waitforupload(fpathaudiofile):
    # Wait for the file to finish uploading
    fsize_a = 0
    fsizecompare = 0
    loopcount = 0

    while fsizecompare == 0:
        fsize_b = os.path.getsize(fpathaudiofile)
        if fsize_b == 0:
            loopcount += 1
            if loopcount == 24:
                changebadfilename(fpathaudiofile)
                return False
            time.sleep(5)
            continue
        if fsize_a == fsize_b:
            fsizecompare = 1
        else:
            fsize_a = fsize_b
        time.sleep(1)
    return True


def checkforaudiofile():
    # Check the directory for files to process
    if os.path.isdir(path):
        files = os.listdir(path)
        for audio in files:
            fpath = path + "/" + audio
            if ((audio.endswith('aax')) and (checkforlock(fpath) is False)):
                if waitforupload(fpath):
                    return audio
                else:
                    return "None"


def createlockfile(fpath):
    # create 0byte file with filename.aax.lock
    lockfile = fpath + ".lock"
    print(f"{time.asctime()} #Creating Lock file: {lockfile}")
    try:
        open(lockfile, "w")
    except Exception as err:
        sys.stderr.write("Couldn't create lock file: {}\n".format(err))
        errorinprocessing = True


def cpfiletoprocessing(fpath, audio):
    # Copy the file to the processing location
    print("Copying the file to processing " + audio)
    try:
        shutil.copy(fpath, "/processing/")
    except Exception as err:
        sys.stderr.write("Couldn't copy the file to processing: {}\n"
                         .format(err)
                         )
        errorinprocessing = True


def processfile(audio):
    # process the file.  Convert from aax to mp3 back on the NAS
    print(f"{time.asctime()} #Processing {audio}")
    command = "./aaxConvert.py -v -d {} /processing/{}\n".format(dpath, audio)
    subprocess.run(command, shell=True)


def copyToArchive(fpath, audio):
    # move aax to archive filesystem
    archive = archivepath + "/" + audio
    try:
        shutil.copy(fpath, archive)
    except Exception as err:
        sys.stderr.write(
            f"{time.asctime()} #Couldn't copy file to the archive: {err}\n"
        )
        errorinprocessing = True


def cleanup(fpath, audio):
    # cleanup after yourself
    #  print"clean up debug: {}".format(fpath)
    # TODO verify md5 hash of archive and original before delete
    archive = archivepath + "/" + audio
    orginial_hash = getMD5(fpath)
    archive_file = getMD5(archive)

    if orginial_hash == archive_file:
        try:
            lockfile = fpath + ".lock"
            print("Removing lock file: " + lockfile)
            os.remove(lockfile)
            os.remove(fpath)
            os.remove('/processing/' + audio)
        except Exception as err:
            sys.stderr.write(
                f"{time.asctime()} Couldn't clean up after myself to the archive: {err}\n"  # noqa E501
            )
            errorinprocessing = True
    else:
        print("Checksum's do not match for fpath.  I am not cleaning up")


def checkforlock(fpathaudiofile):
    # Need to check for .lock file in this function.
    lockfile = fpathaudiofile + ".lock"
    if os.path.exists(lockfile):
        return True
    else:
        return False


def loopforever():
    # main loop for running the app
    while True:
        # print("Looking for files in {}\n").format(path)
        audio = str(checkforaudiofile())
        if audio != "None":
            fpath = path + '/' + audio
            print(f"Debug: {fpath}")
            print(f"{time.asctime()} #Processing file: {audio}\n")
            try:
                createlockfile(fpath)
                cpfiletoprocessing(fpath, audio)  # copy file
                processfile(audio)  # process file

                # Copy file from Incoming to archive
                copyToArchive(fpath, audio)

                cleanup(fpath, audio)  # Clean up the processing dir

                if errorinprocessing is True: # noqa F823
                    print(f"{time.asctime()}There is a error with {audio}\n")

            except Exception:
                continue
            time.sleep(15)
            errorinprocessing = False


if __name__ == "__main__":
    loopforever()

 queryAudiobookServer.py