diff --git a/notebooks/image_search.ipynb b/notebooks/image_search.ipynb new file mode 100644 index 0000000..290a6a2 --- /dev/null +++ b/notebooks/image_search.ipynb @@ -0,0 +1,526 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here we test the implementation of an apk image search functionality with similarity matching. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It requires the installation of the following python packages:\n", + "\n", + " - [parsel](https://github.com/scrapy/parsel) for html parsing\n", + " - [requests](https://github.com/psf/requests), for http requests\n", + " - [fuzzywuzzy](https://github.com/seatgeek/fuzzywuzzy) for title matching\n", + " \n", + " You can install them (assuming you have a python environment with pip) by running this next block:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "!{sys.executable} -m pip install parsel requests fuzzywuzzy" + ] + }, + { + "cell_type": "code", + "execution_count": 290, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import re\n", + "import shutil\n", + "import sys\n", + "import time\n", + "\n", + "import requests\n", + "from fuzzywuzzy import fuzz\n", + "from parsel import Selector\n", + "\n", + "# path to mounted drive\n", + "MOUNT_PATH = \"/tmp/mnt/\"\n", + "# non apk files to ignore\n", + "FOLDER_BLOCKLIST = [\n", + " '.Trash-1000',\n", + " 'APK_packagenames.txt',\n", + " 'badgelist.txt'\n", + "]\n", + "# if the titles in the search results have a lower similarity (0-100) than this threshold ignore them\n", + "MIN_TITLE_THRESHOLD = 70 " + ] + }, + { + "cell_type": "code", + "execution_count": 91, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['.Trash-1000',\n", + " '2MD VR Football Unleashed v1.0.30 -Q2-90Hz-CustomRes -QuestUnderground -steam-1230860 -versionCode-30 -packageName-com.truantpixel.twomdquest',\n", + " 'A Fishermans Tale v1.056 -QuestUnderground -steam-559330 -versionCode-14 -packageName-com.innerspacevr.afishermanstale',\n", + " 'A Lullaby of Colors v1.2.2 -steam-992960 -versionCode-1 -packageName-com.idumpling.a_lullaby_of_colors',\n", + " 'A Moment in Cannes v0.3.22 -steam-252110 -versionCode-1 -packageName-com.azadux.Cannes_Fort',\n", + " 'AFFECTED - The Manor v1.26.1.75 -QuestUnderground -steam-707580 -versionCode-75 -packageName-com.fallen.manorquest',\n", + " 'APK_packagenames.txt',\n", + " 'AUDICA v1.0.3.2.128543601 -QuestUnderground -steam-1020340 -versionCode-128543601 -packageName-com.harmonixmusic.kata',\n", + " 'Accounting+ v1.24.35-Q2Patched -QuestUnderground -steam-927270 -versionCode-35 -packageName-com.crowscrowscrows.AccountingPlus',\n", + " 'Acron - Attack of the Squirrels! v1.14.94392.114094392 -QuestUnderground -steam-1094870 -MP- -versionCode-114094392 -packageName-com.resolutiongames.codenamelazarus',\n", + " 'Air Brigade 2 v0.2 -steam-1163910 -versionCode-20 -packageName-com.OnlyHuman.AirBrigade2vr',\n", + " 'Angry Birds VR Isle of Pigs v3.4.92967.304092967 -Q2-90Hz-CustomRes -QuestUnderground -steam-1001140 -versionCode-304092967 -packageName-com.resolutiongames.abvriop.santacruz',\n", + " 'Apex Construct v45.0 -VRP -steam-694090 -versionCode-45 -packageName-com.fasttravelgames.apexconstruct',\n", + " 'Apollo 11 v2.3.22 -Q2Patched-90Hz-CustomRes -QuestUnderground -steam-457860 -versionCode-22 -packageName-com.immersivevreducation.Apollo11QuestFinal',\n", + " 'Arizona Sunshine v1.5.21549 -QuestUnderground -MP- -steam-342180 -versionCode-21549 -packageName-com.vertigogames.azsq',\n", + " 'ArtPlunge v201909211 -steam-570900 -versionCode-201801250 -packageName-com.spaceplunge.artplungeoc',\n", + " 'Artifact v1.0.6 -steam-821290 -versionCode-17 -packageName-com.Mixreality.Artifact',\n", + " 'Ashtar UFO v1.0 -versionCode-1 -packageName-com.test.mj12 -NA-',\n", + " 'Attack on Quest v0.1.3 -steam-821290 -versionCode-1 -packageName-com.AttaclOnQuest.AttackOnQuestAlpha',\n", + " 'Audio Trip v0.131 -QuestUnderground -steam-826540 -versionCode-33 -packageName-com.KinemotikStudios.AudioTripQuest',\n", + " 'Audioshield v118 -QuestUnderground -steam-412740 -versionCode-118 -packageName-com.Audiosurf.Audioshield',\n", + " 'B-Team v20.03.19.356857 -QuestUnderground -steam-266410 -versionCode-68 -packageName-com.twistedpixelgames.BTEAM_App',\n", + " 'Bait! v1.2 -steam-413150 -versionCode-112063217 -packageName-com.resolutiongames.baitsantacruz',\n", + " 'Ballista v20.42 -steam-1051280 -versionCode-42 -packageName-com.HighVoltage.Crossbow',\n", + " 'BanditSix v1.1.0.130 -steam-357970 -versionCode-130 -packageName-com.climaxstudios.BanditSix',\n", + " 'Bang Squash v1.0 -versionCode-2 -packageName-com.lebersoftware.bangsquashpro -NA-',\n", + " 'Basket VR v0.2 -steam-516900 -versionCode-2 -packageName-com.realdragon.basketlittle',\n", + " 'Beat Saber v1.13.0-147 -VRP (with DLC) -steam-620980 -versionCode-147 -packageName-com.beatgames.beatsaber',\n", + " 'Blair Witch v1.05.33 -QuestUnderground -steam-1092660 -versionCode-33 -packageName-pl.bloober.blairwitch',\n", + " 'Blaston v1.3.96912.103096912 -QuestUnderground -steam-1427890 -MP- -versionCode-103096912 -packageName-com.resolutiongames.ignis',\n", + " 'Blaston v1.4.97724-104097724 -VRP -MP- -steam-1427890 -versionCode-104097724 -packageName-com.resolutiongames.ignis',\n", + " 'Blockami VR v1.00 -versionCode-1 -packageName-com.aardVRk.blockami -NA-',\n", + " 'Bonfire v1.0.466-Q2Patched -QuestUnderground -steam-1155880 -versionCode-466 -packageName-com.baobab.bonfire',\n", + " 'Box VR v1.18.188 -oculus-2327205800645550 -versionCode-188 -packageName-com.fitxr.boxvr',\n", + " 'Captain Hardcore v0.3 -steam-602960 -versionCode-1 -packageName-com.AntiZeroGames.CH',\n", + " 'Car Parking Simulator v1.6.1 -steam-645630 -versionCode-7 -packageName-com.sloppy.carparkingsimulator',\n", + " 'Cat Dissection v1.0 -steam-1178570 -versionCode-1 -packageName-com.VictoryXR.CatDissection',\n", + " 'Cave Digger Riches v2.11-Q2Patched -QuestUnderground -steam-844380 -versionCode-69 -packageName-com.mekiwi.cavedigger',\n", + " 'Chunky v0.9 -steam-535190 -versionCode-1 -packageName-com.BrainBlinks.ChunkyOrbitsQuest',\n", + " 'Chupa Chupa VR v2.0 -VRP -steam-1336160 -versionCode-1 -packageName-com.kan.kikuchi.ChupaChupaVR',\n", + " 'Cirque du Soleil v1.01.13 -steam-712410 -versionCode-1001013 -packageName-com.felixandpaul.quest.cds_portal',\n", + " 'City Avenger v1.0.1 -oculus-1423819361003940 -versionCode-2 -packageName-com.Chesstar.CityAvenger.itch',\n", + " 'Cloudlands 2 v1.31.35 -QuestUnderground -steam-1206960 -versionCode-35 -packageName-com.Futuretown.Cloudlands2',\n", + " 'Coaster Combat [1.0.693251] S2.11 -oculus-3159481670763523 -versionCode-693251 -packageName-com.forcefieldxr.coastercombatquest',\n", + " 'Color Space v1.2.1 -QuestUnderground -steam-244850 -versionCode-251 -packageName-us.lighthaus.colorspace',\n", + " 'Control Tower VR v0.61 -steam-365960 -versionCode-18 -packageName-com.DigitalEmergence.ControlTower_Alpha',\n", + " 'Cook-Out v1.3.97509-103097509 -VRP -oculus-2004774962957063 -versionCode-103097509 -packageName-com.resolutiongames.cookinggame',\n", + " 'Counter Fight v1.5.0 -steam-593210 -versionCode-1 -packageName-jp.tricol.counterfight_for_quest',\n", + " 'Crazy Kung Fu v0.34 -steam-1340300 -versionCode-1 -packageName-com.fieldofvision.crazykungfu',\n", + " 'Creed v1.0-187321-Q2Patched-90Hz -QuestUnderground -steam-804490 -versionCode-187321 -packageName-com.survios.Creed',\n", + " 'Cubism v1.0.5.48 -QuestUnderground -steam-804530 -versionCode-48 -packageName-com.tvb.cubism',\n", + " 'Custom Home Mapper v2.1-beta -VRP -versionCode-1 -packageName-com.curiousvr.homespace -NA-',\n", + " 'Cyber Cycle v0.1 -oculus-2029760787106301 -versionCode-1 -packageName-com.syntheon.cc2',\n", + " 'Dance Central v1.2.1 -QuestUnderground -steam-1331440 -versionCode-121011401 -packageName-com.HarmonixMusic.DCVRQuest',\n", + " 'Dance Dance Maker v0.9 -steam-1294500 -versionCode-9 -packageName-com.novia.vr.ddm',\n", + " 'Dead Shot Heroes v2.1.3 -steam-820480 -versionCode-1 -packageName-com.omm.dsh',\n", + " 'Dead and Buried II [2.0.8396.3008396] S2.11 -oculus-2134077359973067 -versionCode-3008396 -MP- -packageName-com.oculus.dab2',\n", + " 'Death Horizon - Reloaded v0.9.24.92 -QuestUnderground -oculus-2115015981923610 -versionCode-92 -packageName-com.dreamdev.deathhorizon.quest',\n", + " 'Death Lap v9.36.926 -QuestUnderground -steam-1386910 -versionCode-936 -packageName-com.ozwe.deathlap',\n", + " 'Deep Water Solo Climbing v0.1.0 -steam-761890 -versionCode-1 -packageName-com.kecoproductions.DeepWaterSolo',\n", + " 'Deisim v1.31.1 -VRP -steam-525680 -versionCode-30 -packageName-com.MyronSoftware.Deisim',\n", + " 'Descent Alps v0.0.4 -steam-376966 -versionCode-1 -packageName-com.Sutur.DescentAlpsPreRelease',\n", + " 'Disc Benders Ace Run v1.0.0 -versionCode-1 -packageName-com.FreeFallStudios.DiscBenders -NA-',\n", + " 'DiscGolfVR v4.5 -steam-856300 -versionCode-1 -packageName-com.AlexV.DiscGolfVR',\n", + " 'Doctor Who The Edge of Time [368] S2.11 -steam-1075400 -versionCode-368 -packageName-com.mazetheory.doctorwho',\n", + " 'Dogfish Dessection v1.0 -versionCode-1 -packageName-com.VictoryXR.DogfishDissection -NA-',\n", + " 'DrakaVR v1 -versionCode-1 -packageName-com.Questeroid.DrakaVR -NA-',\n", + " 'Dreadhalls v1.6.19.10619-Q2Patched -QuestUnderground -steam-589200 -versionCode-10619 -packageName-com.WhiteDoorGames.DreadhallsQuest',\n", + " 'DroneLight 0.1 -versionCode-1 -packageName-com.dronelight.map1 -NA-',\n", + " 'Drop Dead - Dual Strike Edition v1.3.6.3-Q2Patched -QuestUnderground -oculus-1935017546592600 -versionCode-1724012330 -packageName-com.PixelToys.DropDead',\n", + " 'Drunkn Bar Fight v10.21-Q2Patch90Hz -QuestUnderground -steam-528550 -versionCode-121 -packageName-com.themunky.drunknbarfightQuest',\n", + " 'Dungeon Chess v0.060 -steam-286160 -versionCode-58 -packageName-com.Experiment7.RealDungeonChess',\n", + " 'Dungeon Train v0.19.12 -steam-262060 -versionCode-1 -packageName-com.brennanhatton.dungeontrain',\n", + " 'ENHANCE VR v2.0.0 -steam-1268330 -versionCode-31 -packageName-com.Virtuleap.Enhance',\n", + " 'Electronauts v175667 -QuestUnderground -steam-691160 -versionCode-175667 -packageName-com.survios.Electronauts',\n", + " 'Eleven Table Tennis v1.400 -VRP -steam-488310 -versionCode-400 -packageName-quest.eleven.forfunlabs',\n", + " 'Eleven Table Tennis v1.400b -VRP -steam-488310 -versionCode-400 -packageName-quest.eleven.forfunlabs',\n", + " 'Elven Assassin v1.3.1g-Q2Patch90Hz -QuestUnderground -steam-503770 -versionCode-77 -packageName-com.WenklyStudio.ElvenAssassin',\n", + " 'End Space v1.0.6.22 -QuestUnderground -steam-753900 -versionCode-22 -packageName-com.endspace.quest',\n", + " 'Escape Legacy v1.20 -steam-940300 -versionCode-20 -packageName-com.StormingTech.EscapeLegacySideQuest',\n", + " 'Escape! v1.01 -steam-1145360 -versionCode-2 -packageName-com.Robbin12392.Escape',\n", + " 'Espire 1 VR Operative v1.7.218-1546 -VRP -steam-669290 -versionCode-1546 -packageName-com.DigitalLode.Espire1',\n", + " 'Evryway Visualiser v0.79 -versionCode-1 -packageName-com.Evryway.EvrywayVisualiser -NA-',\n", + " 'Exorcist Legion VR v1.0.9 -QuestUnderground -steam-1156250 -versionCode-27 -packageName-com.wolfandwood.exorcistlegionvr',\n", + " 'FREEDIVER - Triton Down v1.0 -QuestUnderground -steam-995230 -versionCode-20 -packageName-com.Archiact.Freediver',\n", + " 'Face Your Fears 1 and DLC v1.0.536957 (Spoofed as Neverthink) (Q2 only) -QuestUnderground -oculus-1860984660605547 -versionCode-99999999 -packageName-com.neverthink',\n", + " 'Face Your Fears 2 v1.0.487009 -QuestUnderground -oculus-1860984660605547 -versionCode-487009 -packageName-gg.trs.fyf2',\n", + " 'Fail Factory v1.0.54-Q2Patched -QuestUnderground -steam-656510 -versionCode-54 -packageName-com.Armature.FailFactory',\n", + " 'Falcon Age v1.00-8 -QuestUnderground -steam-1075080 -versionCode-8 -packageName-com.OuterloopGames.FalconAge',\n", + " 'Fast Formula v1.0 -steam-1239660 -versionCode-1 -packageName-com.FastFormula.FastFormula',\n", + " 'Five Nights at Freddys Help Wanted v1.0.9-Q2Patched -QuestUnderground -steam-732690 -versionCode-9 -packageName-com.SteelWoolGames.fnafhw',\n", + " 'Food Flinger v1.0 -versionCode-1 -packageName-com.DashingCape.FoodFlinger -NA-',\n", + " 'Frog Dissection v1.0 -steam-1046910 -versionCode-1 -packageName-com.VictoryXR.FrogDissectionQuest',\n", + " 'Fruit Ninja v1.7.0.547288-Q2Patched -QuestUnderground -steam-486780 -versionCode-547288 -packageName-com.halfbrick.fruitninjavr',\n", + " 'Fujii v1.0.825 -QuestUnderground -steam-589040 -versionCode-825 -packageName-com.funktroniclabs.fuji',\n", + " 'Futbolin Revolution v1.0 -steam-1332440 -versionCode-1 -packageName-com.Pixelfun.FutbolinRevolution',\n", + " 'Gadgeteer v1.0.14.76 -QuestUnderground -steam-746560 -versionCode-76 -packageName-com.metanaut.Gadgeteer',\n", + " 'GameList.txt',\n", + " 'Ghost Giant v1.0.32-Q2Patched -QuestUnderground -oculus-2366136696841248 -versionCode-32 -packageName-com.zoink.ghostgiant',\n", + " 'Gladius v0.8.5 (Manual Install see readme) -steam-489630 -versionCode-1 -packageName-com.VirtualAge.GladiusQuest']" + ] + }, + "execution_count": 91, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mount_folders = os.listdir(MOUNT_PATH)\n", + "mount_folders[:100]" + ] + }, + { + "cell_type": "code", + "execution_count": 243, + "metadata": {}, + "outputs": [], + "source": [ + "def parse_folder(folder):\n", + " \"\"\"Removes extra info from the folder name to make search more accurate\"\"\"\n", + " replacement_patterns = [\n", + " '-steam.*',\n", + " '-oculus.*',\n", + " '-versionCode.*',\n", + " '-packageName.*',\n", + " '-MP-.*',\n", + " '-NA-.*',\n", + " '-QuestUnderground.*',\n", + " '-Q2.*',\n", + " r'v(?:(\\d+)\\.?)?(?:(\\d+)\\.?)?(?:(\\d+)\\.?\\d+)\\S*', # version pattern \n", + " ]\n", + "\n", + " for pattern in replacement_patterns:\n", + " folder = re.sub(pattern,'',folder)\n", + " return folder.strip()" + ] + }, + { + "cell_type": "code", + "execution_count": 233, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['2MD VR Football Unleashed',\n", + " 'A Fishermans Tale',\n", + " 'A Lullaby of Colors',\n", + " 'A Moment in Cannes',\n", + " 'AFFECTED - The Manor',\n", + " 'AUDICA',\n", + " 'Accounting+',\n", + " 'Acron - Attack of the Squirrels!',\n", + " 'Air Brigade 2',\n", + " 'Angry Birds VR Isle of Pigs',\n", + " 'Apex Construct -VRP',\n", + " 'Apollo 11',\n", + " 'Arizona Sunshine',\n", + " 'ArtPlunge',\n", + " 'Artifact',\n", + " 'Ashtar UFO',\n", + " 'Attack on Quest',\n", + " 'Audio Trip',\n", + " 'Audioshield',\n", + " 'B-Team',\n", + " 'Bait!',\n", + " 'Ballista',\n", + " 'BanditSix']" + ] + }, + "execution_count": 233, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "parsed_folders = [parse_folder(folder) for folder in mount_folders if folder not in FOLDER_BLOCKLIST]\n", + "parsed_folders[:23]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**SEARCH FUNCTION**\n", + "\n", + "We implement the function to perform a steam store search to get the steam appid" + ] + }, + { + "cell_type": "code", + "execution_count": 292, + "metadata": {}, + "outputs": [], + "source": [ + "def steam_search(search_term, sim_threshold=MIN_TITLE_THRESHOLD):\n", + " \"\"\"\n", + " Finds the most similar steamp appid from a search term\n", + " \"\"\"\n", + " resp = requests.get(\"https://store.steampowered.com/search/\",\n", + " params={'term':search_term, 'vrsupport':1}\n", + " )\n", + " sel = Selector(resp.text)\n", + " results_titles = sel.xpath(\"//span[contains(@class, 'title')]/text()\").extract()\n", + " if not results_titles:\n", + " print(f'NO SEARCH RESULTS FOR TERM {search_term}')\n", + " return\n", + " results_imgs = [img.extract() for img in sel.xpath(\"//div[contains(@class, 'search_capsule')]/img/@src\")]\n", + " results_titles_similarity = [fuzz.partial_ratio(folder, title) for title in results_titles]\n", + " most_similar_id = results_titles_similarity.index(max(results_titles_similarity))\n", + " most_similar_title = results_titles[most_similar_id] \n", + " most_similar_title_similarity = results_titles_similarity[most_similar_id]\n", + " if most_similar_title_similarity < sim_threshold:\n", + " print(f\"NO VALID MATCH FOR TERM '{search_term}'\")\n", + " return\n", + " most_similar_image = results_imgs[most_similar_id] \n", + " appid = re.match(\n", + " '.*/steam/\\w+/(\\d+)/.*jpg', \n", + " most_similar_image).groups(0\n", + " )[0]\n", + " print(f\"FOUND MATCH FOR TERM '{search_term}'->'{most_similar_title}', SIMILARITY:{most_similar_title_similarity} APPID:{appid}\")\n", + " return appid" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can test the search function, we can get a random folder and test a bunch of times" + ] + }, + { + "cell_type": "code", + "execution_count": 293, + "metadata": {}, + "outputs": [], + "source": [ + "import random" + ] + }, + { + "cell_type": "code", + "execution_count": 295, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "FOUND MATCH FOR TERM 'Keep Talking and Nobody Explodes'->'Keep Talking and Nobody Explodes', SIMILARITY:100 APPID:341800\n", + "FOUND MATCH FOR TERM 'The Wizards'->'The Wizards - Dark Times', SIMILARITY:100 APPID:1103860\n", + "NO VALID MATCH FOR TERM 'Together VR (manual install)'\n", + "FOUND MATCH FOR TERM 'Pteranodons Flight - The Flying Dinosaur Game'->'Pteranodon's Flight: The Flying Dinosaur Game', SIMILARITY:96 APPID:1259480\n", + "NO SEARCH RESULTS FOR TERM Void Racer - Extreme\n", + "FOUND MATCH FOR TERM 'Cave Digger Riches'->'Cave Digger VR', SIMILARITY:93 APPID:844380\n", + "FOUND MATCH FOR TERM 'The Line'->'Spec Ops: The Line', SIMILARITY:100 APPID:50300\n", + "FOUND MATCH FOR TERM 'Bait!'->'The Legend of Heroes: Trails of Cold Steel - Shining Pom Bait Value Pack 1', SIMILARITY:80 APPID:605360\n", + "FOUND MATCH FOR TERM 'AFFECTED - The Manor'->'AFFECTED: The Manor', SIMILARITY:89 APPID:707580\n", + "NO VALID MATCH FOR TERM 'Lets Go Chopping'\n" + ] + } + ], + "source": [ + "for _ in range(10):\n", + " time.sleep(1)\n", + " folder = random.choice(parsed_folders)\n", + " steam_search(folder)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**RENAMING FOLDERS**\n", + "\n", + "The final step would be to rename the folders and add the steam id. Since i dont have permissions i will create a bunch of mock folders" + ] + }, + { + "cell_type": "code", + "execution_count": 321, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "mkdir: cannot create directory ‘/tmp/mnt_mock’: File exists\r\n" + ] + } + ], + "source": [ + "!mkdir /tmp/mnt_mock" + ] + }, + { + "cell_type": "code", + "execution_count": 337, + "metadata": {}, + "outputs": [], + "source": [ + "mock_mount_path = '/tmp/mnt_mock/'" + ] + }, + { + "cell_type": "code", + "execution_count": 338, + "metadata": {}, + "outputs": [], + "source": [ + "for folder in mount_folders[:1000]:\n", + " if os.path.exists(f'{mock_mount_path}{folder}'):\n", + " continue\n", + " os.mkdir(f'{mock_mount_path}{folder}')" + ] + }, + { + "cell_type": "code", + "execution_count": 339, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['Journey Of The Gods v1.0.479431-Q2Patched-90Hz -QuestUnderground -oculus-1853479764707533 -versionCode-479431 -packageName-gg.trs.grappa',\n", + " 'Bonfire v1.0.466-Q2Patched -QuestUnderground -steam-1155880 -versionCode-466 -packageName-com.baobab.bonfire',\n", + " 'Void Racer - Extreme v1.08.108 -QuestUnderground -oculus-2874244485968712 -versionCode-108 -MP- -packageName-com.coplanar.vre',\n", + " 'Escape Legacy v1.20 -steam-940300 -versionCode-20 -packageName-com.StormingTech.EscapeLegacySideQuest',\n", + " 'Zombie World VR v1.0 -steam-1206080 -versionCode-4 -packageName-com.Appalga.ZombieWorldVR',\n", + " 'Thumper v1.00 -steam-356400 -versionCode-21 -packageName-com.Drool.Thumper.quest',\n", + " 'Accounting+ v1.24.35-Q2Patched -QuestUnderground -steam-927270 -versionCode-35 -packageName-com.crowscrowscrows.AccountingPlus',\n", + " 'Toy Clash v1.4.0 -steam-620360 -versionCode-42 -packageName-com.fiveminlab.toyclash',\n", + " 'Jigsaw 360 v3.0 -steam-836610 -versionCode-54 -packageName-com.JumbliVR.Jigsaw360',\n", + " 'Phantom - Covert Ops v1.1 -VRP -steam-92700 -versionCode-49401 -packageName-com.nDreams.PhantomQuest']" + ] + }, + "execution_count": 339, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mount_folders = os.listdir(mock_mount_path)\n", + "mount_folders[:10]" + ] + }, + { + "cell_type": "code", + "execution_count": 328, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 332, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Bonfire v1.0.466-Q2Patched -QuestUnderground -steam-1155880 -versionCode-466 -packageName-com.baobab.bonfire'" + ] + }, + "execution_count": 332, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "folder = mount_folders[1]\n", + "folder" + ] + }, + { + "cell_type": "code", + "execution_count": 346, + "metadata": {}, + "outputs": [], + "source": [ + "def rename_folder(folder, mount_path, appid):\n", + " renamed_folder = re.sub('-steam*','',folder) + f' -steam-{appid}'\n", + " print(f'RENAMING {folder}-->{renamed_folder}')\n", + " os.rename(f'{mount_path}{folder}', f'{mount_path}{renamed_folder}')" + ] + }, + { + "cell_type": "code", + "execution_count": 347, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "RENAMING SculptrVR v2.79 -QuestUnderground -steam-418520 -versionCode-162 -packageName-com.sculptrvrinc.sculptrvrapp-->SculptrVR v2.79 -QuestUnderground -418520 -versionCode-162 -packageName-com.sculptrvrinc.sculptrvrapp -steam-12312\n" + ] + } + ], + "source": [ + "appid = '12312'\n", + "rename_folder(folder, mock_mount_path, appid)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def main(mount_path):\n", + " mount_folders = os.listdir(mount_path)[:10]\n", + " parsed_folders = [parse_folder(folder) for folder in mount_folders if folder not in FOLDER_BLOCKLIST]\n", + "\n", + " for folder in parsed_folders:\n", + " time.sleep(1)\n", + " folder = random.choice(parsed_folders)\n", + " appid = steam_search(folder)\n", + " rename_folder(folder, mount_path, appid)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..8769eab --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,3 @@ +requests +parsel +fuzzywuzzy diff --git a/scripts/images.py b/scripts/images.py new file mode 100644 index 0000000..0de4865 --- /dev/null +++ b/scripts/images.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +""" +This script reads folders from a mounted drive, and tries to find the steam appids for the folders +Your python installation requires the packages defined in requirements-dev.txt + +TODO: do something with found appids (rename folders if necessary) +""" + +import os +import re +import sys +import time +import warnings +warnings.simplefilter('ignore') + +import requests +from fuzzywuzzy import fuzz +from parsel import Selector + + +# default path to mounted drive if no arguments are used +DEFAULT_MOUNT_PATH = "/tmp/mnt/" +# non apk files to ignore +FOLDER_BLOCKLIST = [ + '.Trash-1000', + 'APK_packagenames.txt', + 'badgelist.txt', + 'GameList.txt', +] +# if the titles in the search results have a lower similarity (0-100) than this threshold ignore them +MIN_TITLE_THRESHOLD = 70 + +def parse_folder(folder): + """Removes extra info from the folder name to make search more accurate""" + replacement_patterns = [ + '-steam.*', + '-oculus.*', + '-versionCode.*', + '-packageName.*', + '-MP-.*', + '-NA-.*', + '-QuestUnderground.*', + '-Q2.*', + r'v(?:(\d+)\.?)?(?:(\d+)\.?)?(?:(\d+)\.?\d+)\S*', # version pattern + ] + + for pattern in replacement_patterns: + folder = re.sub(pattern,'',folder) + return folder.strip() + + +def steam_search(search_term, sim_threshold=MIN_TITLE_THRESHOLD): + """ + Finds the most similar steamp appid from a search term + """ + resp = requests.get("https://store.steampowered.com/search/", + params={'term':search_term, 'vrsupport':1} + ) + sel = Selector(resp.text) + results_titles = sel.xpath("//span[contains(@class, 'title')]/text()").extract() + if not results_titles: + print(f'NO SEARCH RESULTS FOR TERM {search_term}') + return + results_imgs = [img.extract() for img in sel.xpath("//div[contains(@class, 'search_capsule')]/img/@src")] + results_titles_similarity = [fuzz.partial_ratio(search_term, title) for title in results_titles] + most_similar_id = results_titles_similarity.index(max(results_titles_similarity)) + most_similar_title = results_titles[most_similar_id] + most_similar_title_similarity = results_titles_similarity[most_similar_id] + if most_similar_title_similarity < sim_threshold: + print(f"NO VALID MATCH FOR TERM '{search_term}'") + return + most_similar_image = results_imgs[most_similar_id] + appid = re.match( + '.*/steam/\w+/(\d+)/.*jpg', + most_similar_image).groups(0 + )[0] + print(f"FOUND MATCH FOR TERM '{search_term}'->'{most_similar_title}', SIMILARITY:{most_similar_title_similarity} APPID:{appid}") + return appid + + +def rename_folder(folder, mount_path, appid): + renamed_folder = re.sub('-steam*','',folder) + f' -steam-{appid}' + print(f'RENAMING {folder}-->{renamed_folder}') + os.rename(f'{mount_path}{folder}', f'{mount_path}{renamed_folder}') + + +def main(mount_path): + mount_folders = os.listdir(mount_path) + + for folder in mount_folders: + time.sleep(1) + if folder in FOLDER_BLOCKLIST: + continue + parsed_folder = parse_folder(folder) + appid = steam_search(parsed_folder) + if appid: + rename_folder(folder, mount_path, appid) + + +if __name__=='__main__': + import sys + if len(sys.argv)>1: + mount_path = sys.argv[1] + else: + mount_path = DEFAULT_MOUNT_PATH + main(mount_path)