Update Downloader.bash

This commit is contained in:
RandomNinjaAtk 2023-08-07 11:10:13 +00:00 committed by GitHub
parent 8d054c8c9a
commit 4382c6ab91
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23

View file

@ -8,7 +8,27 @@ source /config/extended.conf
#### Funcitons
log () {
m_time=`date "+%F %T"`
echo $m_time" :: $scriptName :: $scriptVersion :: "$1 2>&1 | tee -a /config/$scriptName.log
}
logfileSetup () {
# auto-clean up log file to reduce space usage
if [ -f "/config/$scriptName.log" ]; then
if find /config -type f -name "$scriptName.log" -size +1024k | read; then
echo "" > /config/$scriptName.log
fi
fi
if [ ! -f "/config/$scriptName.log" ]; then
echo "" > /config/$scriptName.log
chmod 666 "/config/$scriptName.log"
fi
}
UrlDecode () { : "${*//+/ }"; echo -e "${_//%/\\x}"; }
CreatePlatformRomList () {
if [ -f /config/romlist ]; then
rm /config/romlist
@ -16,8 +36,9 @@ CreatePlatformRomList () {
archiveUrl="$(wget -qO- "$1" | grep -io '<a href=['"'"'"][^"'"'"']*['"'"'"]' | sed -e 's/^<a href=["'"'"']//i' -e 's/["'"'"']$//i' | sed 's/\///g' | sort -u | sed "s|^|$1|")"
echo "$archiveUrl" | grep -v "\.\." | sort >> /config/romlist
}
DownloadFileVerification () {
echo "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: Verifing Download..."
log "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: ${fileName} :: Verifing Download..."
case "$1" in
*.zip|*.ZIP)
verify="$(unzip -t "$1" &>/dev/null; echo $?)"
@ -32,20 +53,19 @@ DownloadFileVerification () {
verify="$(chdman verify -i "$1" &>/dev/null; echo $?)"
;;
*.iso|*.ISO|*.hex|*.HEX|*.wasm|*.WASM|*.sv|*.SV)
Log "No methdod to verify this type of file (iso,hex,wasm,sv)"
echo "No methdod to verify this type of file (iso,hex,wasm,sv)"
verify="0"
;;
esac
if [ "$verify" != "0" ]; then
echo "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: ERROR :: Failed Verification!"
log "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: ${fileName} :: ERROR :: Failed Verification!"
rm "$1"
else
echo "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: Download Verified!"
log "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: ${fileName} :: Download Verified!"
fi
}
#### Platforms
PlatformSnes () {
platformName="Super Nintentdo"
@ -57,6 +77,16 @@ PlatformSnes () {
compressRom="false"
}
PlatformMegadrive () {
platformName="Mega Drive"
platformArchiveContentsUrl="https://archive.org/download/retroachievements_collection_v5/Mega%20Drive/"
platformFolder="megadrive"
consoleRomFileExt=".bin, .gen, .md, .sg, .smd, .zip, .7z"
raConsoleId="1"
uncompressRom="false"
compressRom="false"
}
platformsToProcessNumber=0
IFS=',' read -r -a filters <<< "$platforms"
@ -70,17 +100,22 @@ IFS=',' read -r -a filters <<< "$platforms"
for platform in "${filters[@]}"
do
processNumber=$(( $processNumber + 1 ))
echo "$processNumber/$platformToProcessNumber :: $platform"
log "$processNumber/$platformToProcessNumber :: $platform"
if [ $platform == "snes" ]; then
PlatformSnes
elif [ $platform == "megadrive" ]; then
PlatformMegadrive
else
log "ERROR :: No Platforms Selected, exiting..."
exit
fi
echo "$processNumber/$platformToProcessNumber :: $platform :: Finding ROMS..."
log "$processNumber/$platformToProcessNumber :: $platform :: Finding ROMS..."
CreatePlatformRomList "$platformArchiveContentsUrl"
outputdir="$romPath/$platformFolder"
romlist=$(cat /config/romlist)
romListCount=$(echo "$romlist" | wc -l)
echo "$processNumber/$platformToProcessNumber :: $platform :: $romListCount ROMS Found!"
log "$processNumber/$platformToProcessNumber :: $platform :: $romListCount ROMS Found!"
romProcessNumber=0
echo "$romlist" | while read -r rom; do
@ -120,7 +155,7 @@ do
filteredOtherRomsDecoded="$(UrlDecode "$filteredOtherRoms")"
subFolder="$(dirname "$filteredOtherRomsDecoded")"
subFolder="$(basename "$subFolder")"
romUrl=""
if echo "$subFolder" | grep "~" | read; then
subFolder="/$(echo "$subFolder" | cut -d "~" -f 2)/"
else
@ -128,79 +163,58 @@ do
fi
if [ ! -d "${outputdir}${subFolder}" ]; then
echo "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: Creating \"${subFolder}\" folder... "
log "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: Creating \"${subFolder}\" folder... "
mkdir -p "${outputdir}${subFolder}"
chmod 777 "${outputdir}${subFolder}"
fi
echo "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: Searching ROM Folder"
log "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: Searching Archive URL ROM Folder"
if [ $filteredUsaRomscount -eq 1 ]; then
echo "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: USA (U) ROM FOUND"
log "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: USA (U) ROM FOUND"
fileName="$(basename "$filteredUsaRoms")"
fileName="$(UrlDecode "$fileName")"
echo "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: ${outputdir}${subFolder}${fileName}"
if [ ! -f "${outputdir}${subFolder}${fileName}" ]; then
echo "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: Downloading \"${fileName}\" ROM..."
wget "$filteredUsaRoms" -O "${outputdir}${subFolder}${fileName}"
DownloadFileVerification "${outputdir}${subFolder}${fileName}"
fi
romUrl="$filteredUsaRoms"
elif [ $filteredUsa2Romscount -eq 1 ]; then
echo "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: USA (USA) ROM FOUND"
log "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: USA (USA) ROM FOUND"
fileName="$(basename "$filteredUsa2Roms")"
fileName="$(UrlDecode "$fileName")"
echo "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: ${outputdir}${subFolder}${fileName}"
if [ ! -f "${outputdir}${subFolder}${fileName}" ]; then
echo "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: Downloading \"${fileName}\" ROM..."
wget "$filteredEuropRoms" -O "${outputdir}${subFolder}${fileName}"
DownloadFileVerification "${outputdir}${subFolder}${fileName}"
fi
romUrl="$filteredUsa2Roms"
elif [ $filteredEuropRomscount -eq 1 ]; then
echo "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: EUROPE ROM FOUND"
log "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: EUROPE ROM FOUND"
fileName="$(basename "$filteredEuropRoms")"
fileName="$(UrlDecode "$fileName")"
echo "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: ${outputdir}${subFolder}${fileName}"
if [ ! -f "${outputdir}${subFolder}${fileName}" ]; then
echo "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: Downloading \"${fileName}\" ROM..."
wget "$filteredEuropRoms" -O "${outputdir}${subFolder}${fileName}"
DownloadFileVerification "${outputdir}${subFolder}${fileName}"
fi
romUrl="$filteredEuropRoms"
elif [ $filteredWorldRomscount -eq 1 ]; then
echo "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: WORLD ROM FOUND"
log "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: WORLD ROM FOUND"
fileName="$(basename "$filteredWorldRoms")"
fileName="$(UrlDecode "$fileName")"
echo "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: ${outputdir}${subFolder}${fileName}"
if [ ! -f "${outputdir}${subFolder}${fileName}" ]; then
echo "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: Downloading \"${fileName}\" ROM..."
wget "$filteredWorldRoms" -O "${outputdir}${subFolder}${fileName}"
DownloadFileVerification "${outputdir}${subFolder}${fileName}"
fi
romUrl="$filteredWorldRoms"
elif [ $filteredJapanRomscount -eq 1 ]; then
echo "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: JAPAN ROM FOUND"
log "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: JAPAN ROM FOUND"
fileName="$(basename "$filteredJapanRoms")"
fileName="$(UrlDecode "$fileName")"
echo "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: ${outputdir}${subFolder}${fileName}"
if [ ! -f "${outputdir}${subFolder}${fileName}" ]; then
echo "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: Downloading \"${fileName}\" ROM..."
wget "$filteredJapanRoms" -O "${outputdir}${subFolder}${fileName}"
DownloadFileVerification "${outputdir}${subFolder}${fileName}"
fi
romUrl="$filteredJapanRoms"
elif [ $filteredOtherRomscount -eq 1 ]; then
echo "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: OTHER ROM FOUND"
log "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: OTHER ROM FOUND"
fileName="$(basename "$filteredOtherRoms")"
fileName="$(UrlDecode "$fileName")"
echo "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: ${outputdir}${subFolder}${fileName}"
if [ ! -f "${outputdir}${subFolder}${fileName}" ]; then
echo "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: Downloading \"${fileName}\" ROM..."
wget "$filteredOtherRoms" -O "${outputdir}${subFolder}${fileName}"
DownloadFileVerification "${outputdir}${subFolder}${fileName}"
fi
romUrl="$filteredOtherRoms"
else
echo "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: ERROR :: No Filtered Roms Found..."
log "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: ERROR :: No Filtered Roms Found..."
continue
fi
if [ ! -f "${outputdir}${subFolder}${fileName}" ]; then
log "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: ${fileName} :: ROM downloading to \"${outputdir}${subFolder}\"..."
wget "$romUrl" -O "${outputdir}${subFolder}${fileName}"
else
log "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: ${fileName} :: ROM previously downloaded..."
fi
if [ -f "${outputdir}${subFolder}${fileName}" ]; then
echo "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: Setting Permissions on: ${fileName}"
DownloadFileVerification "${outputdir}${subFolder}${fileName}"
log "$processNumber/$platformToProcessNumber :: $platform :: $romProcessNumber/$romListCount :: ${fileName} :: Setting Permissions to 666"
chmod 666 "${outputdir}${subFolder}${fileName}"
fi
@ -208,175 +222,8 @@ do
rm /config/romfilelist
fi
done
done
exit
url="https://archive.org/download/retroachievements_collection_v5/SNES/Super%20Mario%20Kart/Super%20Mario%20Kart%20%28U%29%20%5B%21%5D.zip"
decodedUrl="$(UrlDecode "$url")"
echo $decodedUrl
url="https://archive.org/download/retroachievements_collection_v5/SNES"
decodedUrl="$(UrlDecode "$url")"
echo $decodedUrl
if [ -f romlist ]; then
rm romlist
fi
if [ -f romfilelist ]; then
rm romfilelist
fi
outputdir="/home/steveno/Nextcloud/Gaming/ROMs/megadrive"
archiveContentsUrl="https://archive.org/download/retroachievements_collection_v5/SNES/"
archiveContentsUrl="https://archive.org/download/retroachievements_collection_v5/NES/"
archiveContentsUrl="https://archive.org/download/retroachievements_collection_v5/Mega%20Drive/"
archiveUrl="$(wget -qO- "$archiveContentsUrl" | grep -io '<a href=['"'"'"][^"'"'"']*['"'"'"]' | sed -e 's/^<a href=["'"'"']//i' -e 's/["'"'"']$//i' | sed 's/\///g' | sort -u | sed "s|^|$archiveContentsUrl|")"
echo "$archiveUrl" | grep -v "\.\." | sort >> romlist
romlist=$(cat romlist)
echo "$romlist" | while read -r rom; do
archiveContentsUrl="$rom/"
archiveUrl="$(wget -qO- "$archiveContentsUrl" | grep -i ".zip" | grep -io '<a href=['"'"'"][^"'"'"']*['"'"'"]' | sed -e 's/^<a href=["'"'"']//i' -e 's/["'"'"']$//i' | sed 's/\///g' | sort -u | sed "s|^|$archiveContentsUrl|")"
echo "$archiveUrl" >> romfilelist
romfiles="$(cat romfilelist | awk '{ print length, $0 }' | sort -n | cut -d" " -f2-)"
# debugging
#echo "original list: "
#cat romfilelist
#echo ""
#echo "rom file list sorted by length: "
#echo "$romfiles"
#filteredUsaRoms="$(echo "$romfiles" | grep "%20%28U%29" | head -n 1)"
#echo ""
#echo "filtered:"
#echo "$filteredUsaRoms"
#if [ -f romfilelist ]; then
# rm romfilelist
#fi
#continue\
filteredUsaRoms="$(echo "$romfiles" | grep -i "%20%28U%29" | head -n 1)"
filteredUsaRomscount="$(echo "$romfiles" | grep -i "%20%28U%29" | head -n 1 | wc -l)"
filteredUsa2Roms="$(echo "$romfiles" | grep -i "%20%28USA%29" | head -n 1)"
filteredUsa2Romscount="$(echo "$romfiles" | grep -i "%20%28USA%29" | head -n 1 | wc -l)"
filteredEuropRoms="$(echo "$romfiles" | grep -i "%20%28E%29" | head -n 1)"
filteredEuropRomscount="$(echo "$romfiles" | grep -i "%20%28E%29" | head -n 1 | wc -l)"
filteredWorldRoms="$(echo "$romfiles" | grep -i "%20%28W%29" | head -n 1)"
filteredWorldRomscount="$(echo "$romfiles" | grep -i "%20%28W%29" | head -n 1 | wc -l)"
filteredJapanRoms="$(echo "$romfiles" | grep -i "%20%28J%29" | head -n 1)"
filteredJapanRomscount="$(echo "$romfiles" | grep -i "%20%28J%29" | head -n 1 | wc -l)"
filteredOtherRoms="$(echo "$romfiles" | head -n 1)"
filteredOtherRomscount="$(echo "$romfiles" | head -n 1 | wc -l)"
filteredOtherRomsDecoded="$(UrlDecode "$filteredOtherRoms")"
subFolder="$(dirname "$filteredOtherRomsDecoded")"
subFolder="$(basename "$subFolder")"
if echo "$subFolder" | grep "~" | read; then
subFolder="/$(echo "$subFolder" | cut -d "~" -f 2)/"
else
subFolder="/"
fi
if [ ! -d "${outputdir}${subFolder}" ]; then
echo "Creating \"${subFolder}\" folder... "
mkdir -p "${outputdir}${subFolder}"
chmod 777 "${outputdir}${subFolder}"
fi
if [ $filteredUsaRomscount -eq 1 ]; then
echo "USA (U) ROM FOUND"
fileName="$(basename "$filteredUsaRoms")"
fileName="$(UrlDecode "$fileName")"
echo "${outputdir}${subFolder}${fileName}"
if [ ! -f "${outputdir}${subFolder}${fileName}" ]; then
wget "$filteredUsaRoms" -O "${outputdir}${subFolder}${fileName}"
DownloadFileVerification "${outputdir}${subFolder}${fileName}"
fi
elif [ $filteredUsa2Romscount -eq 1 ]; then
echo "USA (USA) ROM FOUND"
fileName="$(basename "$filteredUsa2Roms")"
fileName="$(UrlDecode "$fileName")"
echo "${outputdir}${subFolder}${fileName}"
if [ ! -f "${outputdir}${subFolder}${fileName}" ]; then
wget "$filteredEuropRoms" -O "${outputdir}${subFolder}${fileName}"
DownloadFileVerification "${outputdir}${subFolder}${fileName}"
fi
elif [ $filteredEuropRomscount -eq 1 ]; then
echo "EUROPE ROM FOUND"
fileName="$(basename "$filteredEuropRoms")"
fileName="$(UrlDecode "$fileName")"
echo "${outputdir}${subFolder}${fileName}"
if [ ! -f "${outputdir}${subFolder}${fileName}" ]; then
wget "$filteredEuropRoms" -O "${outputdir}${subFolder}${fileName}"
DownloadFileVerification "${outputdir}${subFolder}${fileName}"
fi
elif [ $filteredWorldRomscount -eq 1 ]; then
echo "WORLD ROM FOUND"
fileName="$(basename "$filteredWorldRoms")"
fileName="$(UrlDecode "$fileName")"
echo "${outputdir}${subFolder}${fileName}"
if [ ! -f "${outputdir}${subFolder}${fileName}" ]; then
wget "$filteredWorldRoms" -O "${outputdir}${subFolder}${fileName}"
DownloadFileVerification "${outputdir}${subFolder}${fileName}"
fi
elif [ $filteredJapanRomscount -eq 1 ]; then
echo "JAPAN ROM FOUND"
fileName="$(basename "$filteredJapanRoms")"
fileName="$(UrlDecode "$fileName")"
echo "${outputdir}${subFolder}${fileName}"
if [ ! -f "${outputdir}${subFolder}${fileName}" ]; then
wget "$filteredJapanRoms" -O "${outputdir}${subFolder}${fileName}"
DownloadFileVerification "${outputdir}${subFolder}${fileName}"
fi
elif [ $filteredOtherRomscount -eq 1 ]; then
echo "OTHER ROM FOUND"
fileName="$(basename "$filteredOtherRoms")"
fileName="$(UrlDecode "$fileName")"
echo "${outputdir}${subFolder}${fileName}"
if [ ! -f "${outputdir}${subFolder}${fileName}" ]; then
wget "$filteredOtherRoms" -O "${outputdir}${subFolder}${fileName}"
DownloadFileVerification "${outputdir}${subFolder}${fileName}"
fi
else
echo "ERROR :: No Filtered Roms Found..."
fi
if [ -f "${outputdir}${subFolder}${fileName}" ]; then
echo "Setting Permissions on: ${fileName}"
chmod 666 "${outputdir}${subFolder}${fileName}"
fi
if [ -f romfilelist ]; then
rm romfilelist
fi
downloadedRomCount=$(find "$outputdir" -type f | wc -l)
log "$processNumber/$platformToProcessNumber :: $platform :: $downloadedRomCount ROMS Successfully Downloaded!!"
done
exit
archiveContentsUrl="https://archive.org/download/retroachievements_collection_v5/SNES/World Heroes/"
archiveUrl="$(wget -qO- "$archiveContentsUrl" | grep -i ".zip" | grep -io '<a href=['"'"'"][^"'"'"']*['"'"'"]' | sed -e 's/^<a href=["'"'"']//i' -e 's/["'"'"']$//i' | sed 's/\///g' | sort -u | sed "s|^|$archiveContentsUrl|")"
echo "$(UrlDecode "$archiveUrl")" >> romfilelist
romfiles=$(cat romfilelist)
echo "$romfiles"
echo ""
if [ -f romlist ]; then
rm romlist
fi
if [ -f romfilelist ]; then
rm romfilelist
fi
exit
#curl -s "https://archive.org/download/retroachievements_collection_v5/SNES/Super%20Mario%20Kart/Super%20Mario%20Kart%20%28U%29%20%5B%21%5D.zip"