This question has been asked here a few times and I've read and tried most of the answers, but I'm no closer to the solution.
When I enter the URL manually in a browser (I have tried Firefox, Edge and Opera) everything works fine.
As soon as I start my script, I get a: "urllib.error.HTTPError: HTTP Error 403: Forbidden"
Originally I had not set a header in the request, but now the header is in and I still get a 403.Line: "with urllib.request.urlopen(f"{MP_URL}/api/assets/asset/{el['id']}") as f:"
Code:
import jsonimport osimport urllib.requestopener = urllib.request.build_opener()opener.addheaders = [('User-Agent', 'Mozilla 5.0')]urllib.request.install_opener(opener)MP_URL = "https://www.unrealengine.com/marketplace"JSON_DIR = "./data"OUT_FILE = "mp_vault_list.md"def main(): mp_jsons = [] cost = 0. disc = 0. currency = "" count = 0 for f in os.listdir(JSON_DIR): if not f.endswith(".json"): continue with open(os.path.join(JSON_DIR, f), encoding="utf8") as fd: j = json.load(fd) count += len(j['data']['elements']) mp_jsons.append(j) with open(OUT_FILE, "w+", encoding="utf8") as fd: fd.write(f"# Vault content\n\n") i = 0 for j in mp_jsons: for el in j['data']['elements']: i += 1 print(f"\rProcessing Vault {i}/{count}", end="") # The ID found in the json dump is completely valid, and we have to resolve it to the urlSlug if we # want a nice and valid URL with urllib.request.urlopen(f"{MP_URL}/api/assets/asset/{el['id']}") as f: jj = json.load(f) jjd = jj['data']['data'] fd.write(f"* [{jjd['title']}]({MP_URL}/en-US/product/{jjd['urlSlug']})\n") cost += jjd['priceValue'] currency = jjd['currencyCode'] try: disc += jjd['discountPriceValue'] except KeyError: pass fd.write(f"\n# Vault stats\n\n") fd.write(f"* Value: {cost / 100}{currency}\n") fd.write(f"* Current cost: {disc / 100}{currency}\n") fd.write(f"* Size: {count}\n")if __name__ == '__main__': main()Originally I had not set a header in the request, but now the header is in and I still get a 403.
JSON file in folder data:
{"status": "OK","data": {"elements": [ {"catalogItemId": "c1889d04a19f42cd86341de462bdd64b","namespace": "ue","categories": null,"title": "Tappy Chicken","description": "Tappy Chicken demonstrates how to create a 16-bit-style side-scrolling game using Blueprint!","seller": {"id": "o-aa83a0a9bc45e98c80c1b1c9d92e9e","noAi": false,"owner": "311cffd7000046919e47765426990332","status": "ACTIVE","accepted": [ {"ns": "ue","custom": false } ],"financeCheckExempted": true,"name": "Epic Games","supportEmail": "marketplace-support@unrealengine.com" },"keyImages": [ {"type": "Thumbnail","url": "https://cdn1.epicgames.com/ue/item/store_TappyChicken_thumbnail-140x140-5f29f37cd10f728c6aaef67b811f2085.png","md5": "5f29f37cd10f728c6aaef67b811f2085","width": 140,"height": 140,"size": 18856,"uploadedDate": "2014-07-31T20:00:16.640Z" }, {"type": "Screenshot","url": "https://cdn1.epicgames.com/ue/item/store_TappyChicken_screenshot_3-1920x1080-0315ae3bfe61ea5a124dca24274620fc.png","md5": "0315ae3bfe61ea5a124dca24274620fc","width": 1920,"height": 1080,"size": 225338,"uploadedDate": "2014-07-31T20:00:33.092Z" }, {"type": "Screenshot","url": "https://cdn1.epicgames.com/ue/item/store_TappyChicken_screenshot_1-1920x1080-797505e1581ee9a86a55b3a88b40c28c.png","md5": "797505e1581ee9a86a55b3a88b40c28c","width": 1920,"height": 1080,"size": 131210,"uploadedDate": "2014-07-31T20:00:23.384Z" }, {"type": "Featured","url": "https://cdn1.epicgames.com/ue/item/store_TappyChicken_featured-476x246-0ff729b32eac8911749b350a02c26178.png","md5": "0ff729b32eac8911749b350a02c26178","width": 476,"height": 246,"size": 30643,"uploadedDate": "2014-07-31T20:00:05.976Z" }, {"type": "Screenshot","url": "https://cdn1.epicgames.com/ue/item/store_TappyChicken_screenshot_2-1920x1080-191ad0f829e90e1884bae1dad4fa5bcd.png","md5": "191ad0f829e90e1884bae1dad4fa5bcd","width": 1920,"height": 1080,"size": 240490,"uploadedDate": "2014-07-31T20:00:45.565Z" }, {"type": "Screenshot","url": "https://cdn1.epicgames.com/ue/item/store_TappyChicken_screenshot_4-1920x1080-483f24f2a6672156176b16b17e29ae64.png","md5": "483f24f2a6672156176b16b17e29ae64","width": 1920,"height": 1080,"size": 56107,"uploadedDate": "2014-07-31T20:00:55.678Z" } ],"thumbnail": "https://cdn1.epicgames.com/ue/item/store_TappyChicken_thumbnail-140x140-5f29f37cd10f728c6aaef67b811f2085.png","learnThumbnail": "https://cdn1.epicgames.com/ue/item/store_TappyChicken_featured-476x246-0ff729b32eac8911749b350a02c26178.png","bundle": false,"owned": true,"downloadable": true,"isCatalogItem": true,"commentRatingId": "c1889d04a19f42cd86341de462bdd64b","id": "6af75ba1005e41d0bdacd277f6acf3a2","ratingId": "6af75ba1005e41d0bdacd277f6acf3a2","rating": {"targetId": "6af75ba1005e41d0bdacd277f6acf3a2","averageRating": 0,"rating5": 0,"rating4": 0,"rating3": 0,"rating2": 0,"rating1": 0,"legacyRatingNum": 2,"rating5Percent": 0,"rating4Percent": 0,"rating3Percent": 0,"rating2Percent": 0,"rating1Percent": 0,"total": 0 },"reviewed": false } ] }}