This question has been asked here a few times and I've read and tried most of the answers, but I'm no closer to the solution.
When I enter the URL manually in a browser (I have tried Firefox, Edge and Opera) everything works fine.
As soon as I start my script, I get a: "urllib.error.HTTPError: HTTP Error 403: Forbidden"
Originally I had not set a header in the request, but now the header is in and I still get a 403.Line: "with urllib.request.urlopen(f"{MP_URL}/api/assets/asset/{el['id']}") as f:"
Code:
import jsonimport osimport urllib.requestopener = urllib.request.build_opener()opener.addheaders = [('User-Agent', 'Mozilla 5.0')]urllib.request.install_opener(opener)MP_URL = "https://www.unrealengine.com/marketplace"JSON_DIR = "./data"OUT_FILE = "mp_vault_list.md"def main(): mp_jsons = [] cost = 0. disc = 0. currency = "" count = 0 for f in os.listdir(JSON_DIR): if not f.endswith(".json"): continue with open(os.path.join(JSON_DIR, f), encoding="utf8") as fd: j = json.load(fd) count += len(j['data']['elements']) mp_jsons.append(j) with open(OUT_FILE, "w+", encoding="utf8") as fd: fd.write(f"# Vault content\n\n") i = 0 for j in mp_jsons: for el in j['data']['elements']: i += 1 print(f"\rProcessing Vault {i}/{count}", end="") # The ID found in the json dump is completely valid, and we have to resolve it to the urlSlug if we # want a nice and valid URL with urllib.request.urlopen(f"{MP_URL}/api/assets/asset/{el['id']}") as f: jj = json.load(f) jjd = jj['data']['data'] fd.write(f"* [{jjd['title']}]({MP_URL}/en-US/product/{jjd['urlSlug']})\n") cost += jjd['priceValue'] currency = jjd['currencyCode'] try: disc += jjd['discountPriceValue'] except KeyError: pass fd.write(f"\n# Vault stats\n\n") fd.write(f"* Value: {cost / 100}{currency}\n") fd.write(f"* Current cost: {disc / 100}{currency}\n") fd.write(f"* Size: {count}\n")if __name__ == '__main__': main()Originally I had not set a header in the request, but now the header is in and I still get a 403.
JSON file in folder data:{"status":"OK","data":{"elements":[{"catalogItemId":"c1889d04a19f42cd86341de462bdd64b","namespace":"ue","categories":null,"title":"Tappy Chicken","description":"Tappy Chicken demonstrates how to create a 16-bit-style side-scrolling game using Blueprint!","seller":{"id":"o-aa83a0a9bc45e98c80c1b1c9d92e9e","noAi":false,"owner":"311cffd7000046919e47765426990332","status":"ACTIVE","accepted":[{"ns":"ue","custom":false}],"financeCheckExempted":true,"name":"Epic Games","supportEmail":"marketplace-support@unrealengine.com"},"keyImages":[{"type":"Thumbnail","url":"https://cdn1.epicgames.com/ue/item/store_TappyChicken_thumbnail-140x140-5f29f37cd10f728c6aaef67b811f2085.png","md5":"5f29f37cd10f728c6aaef67b811f2085","width":140,"height":140,"size":18856,"uploadedDate":"2014-07-31T20:00:16.640Z"},{"type":"Screenshot","url":"https://cdn1.epicgames.com/ue/item/store_TappyChicken_screenshot_3-1920x1080-0315ae3bfe61ea5a124dca24274620fc.png","md5":"0315ae3bfe61ea5a124dca24274620fc","width":1920,"height":1080,"size":225338,"uploadedDate":"2014-07-31T20:00:33.092Z"},{"type":"Screenshot","url":"https://cdn1.epicgames.com/ue/item/store_TappyChicken_screenshot_1-1920x1080-797505e1581ee9a86a55b3a88b40c28c.png","md5":"797505e1581ee9a86a55b3a88b40c28c","width":1920,"height":1080,"size":131210,"uploadedDate":"2014-07-31T20:00:23.384Z"},{"type":"Featured","url":"https://cdn1.epicgames.com/ue/item/store_TappyChicken_featured-476x246-0ff729b32eac8911749b350a02c26178.png","md5":"0ff729b32eac8911749b350a02c26178","width":476,"height":246,"size":30643,"uploadedDate":"2014-07-31T20:00:05.976Z"},{"type":"Screenshot","url":"https://cdn1.epicgames.com/ue/item/store_TappyChicken_screenshot_2-1920x1080-191ad0f829e90e1884bae1dad4fa5bcd.png","md5":"191ad0f829e90e1884bae1dad4fa5bcd","width":1920,"height":1080,"size":240490,"uploadedDate":"2014-07-31T20:00:45.565Z"},{"type":"Screenshot","url":"https://cdn1.epicgames.com/ue/item/store_TappyChicken_screenshot_4-1920x1080-483f24f2a6672156176b16b17e29ae64.png","md5":"483f24f2a6672156176b16b17e29ae64","width":1920,"height":1080,"size":56107,"uploadedDate":"2014-07-31T20:00:55.678Z"}],"thumbnail":"https://cdn1.epicgames.com/ue/item/store_TappyChicken_thumbnail-140x140-5f29f37cd10f728c6aaef67b811f2085.png","learnThumbnail":"https://cdn1.epicgames.com/ue/item/store_TappyChicken_featured-476x246-0ff729b32eac8911749b350a02c26178.png","bundle":false,"owned":true,"downloadable":true,"isCatalogItem":true,"commentRatingId":"c1889d04a19f42cd86341de462bdd64b","id":"6af75ba1005e41d0bdacd277f6acf3a2","ratingId":"6af75ba1005e41d0bdacd277f6acf3a2","rating":{"targetId":"6af75ba1005e41d0bdacd277f6acf3a2","averageRating":0,"rating5":0,"rating4":0,"rating3":0,"rating2":0,"rating1":0,"legacyRatingNum":2,"rating5Percent":0,"rating4Percent":0,"rating3Percent":0,"rating2Percent":0,"rating1Percent":0,"total":0},"reviewed":false}]}}enter code hereenter code here