#!/usr/bin/env python3 """Recover 90 archived Notion entries by unarchiving, crawling source URLs, and setting covers.""" import json, os, subprocess, sys, time, re, urllib.parse, hashlib import requests API_KEY = "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" HEADERS = { "Authorization": f"Bearer {API_KEY}", "Notion-Version": "2022-06-28", "Content-Type": "application/json" } COVER_DIR = "/root/.openclaw/workspace/public/taste/covers/recovered" TASTE_DNA_DIR = "/root/.openclaw/workspace/public/visual-research/taste-dna" BASE_URL = "https://www.curiousendeavor.com/taste/covers/recovered" PAGE_IDS = """318330c2-8646-8188-8779-d61bc958917f 318330c2-8646-81d2-ae5f-e19d3defcf20 30e330c2-8646-8106-ae31-d41fa54d5bb1 30e330c2-8646-8116-94e0-dc9d75644e89 30e330c2-8646-811c-b57a-f2e8fc09b7d6 30e330c2-8646-8124-94e3-fe4722c5161e 30e330c2-8646-8129-8629-ce754dd0408c 30e330c2-8646-8132-adeb-ec9e7f8650c6 30e330c2-8646-81af-9864-c3cb58f1c33b 30e330c2-8646-81cd-adeb-e39022424d7e 2ff330c2-8646-810b-8230-d5afa1588e48 2ff330c2-8646-8112-80a7-c06d5ada2cca 2ff330c2-8646-8112-9b83-e039501b41af 2ff330c2-8646-8111-b8ef-e482f3d839fd 2ff330c2-8646-814a-ae6e-fbb6096e1447 2ff330c2-8646-81a1-9564-c37fc50fc806 2ff330c2-8646-811b-a667-d03b1d508d2c 2ff330c2-8646-81e4-a6e6-e38738c8c0c2 2ff330c2-8646-8131-b827-fb81c03de895 2ff330c2-8646-8125-8389-cc985cdd5584 2ff330c2-8646-8152-b494-dd2b3b14ddc0 2ff330c2-8646-815e-a075-d779e84e98b4 2ff330c2-8646-81cc-9ead-ebe566aebbd6 2ff330c2-8646-815b-9e95-db1003c670b7 2ff330c2-8646-81a8-9025-e3c635e093da 2ff330c2-8646-81d5-afa7-fffa50ad8803 2ff330c2-8646-81d6-be33-e5830e6aa3ae 2ff330c2-8646-81f9-9189-e0341a9e670a 2ff330c2-8646-810c-9084-d5c2e5ca9cc2 2ff330c2-8646-8134-8cb0-faf7d3e93531 2ff330c2-8646-8138-856d-df1c8fc2c082 2ff330c2-8646-8152-a290-d2d9790881bb 2ff330c2-8646-8159-97a0-d8d8d749ca18 2ff330c2-8646-8161-af2c-f528dc2c950e 2ff330c2-8646-816b-bdcd-d37a44290e04 2ff330c2-8646-8189-af18-e06eef039718 2ff330c2-8646-8190-9dc3-efbdab3cfa76 2ff330c2-8646-8194-8cf7-fa2739940d7a 2ff330c2-8646-8198-b356-d7f43ca183d9 2ff330c2-8646-819a-946c-dd63d540b0cc 2ff330c2-8646-81e0-a038-d26975259489 2ff330c2-8646-81eb-a40e-c560e166f71c 2ff330c2-8646-81ef-a0b6-e8f7df69e3a7 2ff330c2-8646-8109-9a0d-dd0c0d236e2e 2ff330c2-8646-8169-a59f-fe6000c15835 2ff330c2-8646-817d-9158-d10158001458 2ff330c2-8646-817d-abfe-cea1ddf7def4 2ff330c2-8646-8181-9aec-fa90e58bb8d1 2ff330c2-8646-818c-9569-d98df249d5b9 2ff330c2-8646-8193-b70c-ccf4154e9b4e 2ff330c2-8646-81a5-a659-f11ae9e5e461 2ff330c2-8646-81ba-8a44-d066f765ce9d 2ff330c2-8646-81bf-913c-efe51fdbe9b1 2ff330c2-8646-81c7-85fc-c865a4abffd7 2ff330c2-8646-81e7-b410-fa9857fbf7f5 2ff330c2-8646-81f6-88d1-ec2c30d6db2a 2ff330c2-8646-810b-84af-ebd6cb07e764 2ff330c2-8646-8115-9dbe-dc7140c1a94c 2ff330c2-8646-8127-906c-ce39d91e103e 2ff330c2-8646-812a-a04e-f9cd21aa00b2 2ff330c2-8646-812c-a16b-f204ec6d28a4 2ff330c2-8646-8155-8640-da8552206f4a 2ff330c2-8646-8173-a3cf-eae8c31a537e 2ff330c2-8646-81a7-a55f-d98c4e2d15e6 2ff330c2-8646-81d5-8990-c7a63094822f 2ff330c2-8646-8188-8455-fe1dc73a212c 2ff330c2-8646-818a-babb-d5455414ad71 2ff330c2-8646-81dd-967f-f87086d6d03e 2ff330c2-8646-81df-989f-fddef2b315e8 2ff330c2-8646-81fb-aad5-c6e24059ada3 2ff330c2-8646-81fd-aa83-c0748459a20d 2ff330c2-8646-810b-a8aa-e690178e4678 2ff330c2-8646-8113-abf9-f511696b3f95 2ff330c2-8646-8125-aa4a-f8f2d8311497 2ff330c2-8646-8159-90c5-e9f984cf6f23 2ff330c2-8646-8192-ba0c-d757cf4d5da5 2ff330c2-8646-8195-83f2-ea55fd63ed9c 2ff330c2-8646-81af-a9e0-c55aa48d4b1c 2ff330c2-8646-81e7-9a00-c792db85635a 2ff330c2-8646-81f8-af57-e15e2c5954c2 2ff330c2-8646-8119-8156-fa5b1fb7889e 2ff330c2-8646-812b-b934-da7ddabecc61 2ff330c2-8646-812f-ba33-eb5740b53306 2ff330c2-8646-8139-abe6-e13dc586a8a3 2ff330c2-8646-815b-ac4a-e8ba0c72a6c6 2ff330c2-8646-81d5-9efa-eed126c948d9 2ff330c2-8646-81f1-bcf7-dc990689a769 2ff330c2-8646-81f6-9d76-e41f8de30f2e 2ff330c2-8646-8100-a484-c4e37df1d6a7 2ff330c2-8646-8134-a379-eb11d1986b9e""".strip().split("\n") def slugify(text): text = text.lower().strip() text = re.sub(r'[^\w\s-]', '', text) text = re.sub(r'[\s_]+', '-', text) text = re.sub(r'-+', '-', text) return text[:80].strip('-') def decode_html_entities(url): """Decode HTML entities like / -> /""" import html return html.unescape(url) def extract_screenshotone_url(url): """Extract target URL from ScreenshotOne API URL""" parsed = urllib.parse.urlparse(url) params = urllib.parse.parse_qs(parsed.query) if 'url' in params: return params['url'][0] return None def get_title(page): props = page.get("properties", {}) for key in ["Name", "Title", "title"]: if key in props: title_arr = props[key].get("title", []) if title_arr: return "".join(t.get("plain_text", "") for t in title_arr) return "untitled" def get_link(page): props = page.get("properties", {}) if "Link" in props: url_obj = props["Link"].get("url") return url_obj return None def unarchive_page(page_id): r = requests.patch( f"https://api.notion.com/v1/pages/{page_id}", headers=HEADERS, json={"archived": False} ) return r.status_code == 200 def set_cover(page_id, cover_url): r = requests.patch( f"https://api.notion.com/v1/pages/{page_id}", headers=HEADERS, json={"cover": {"type": "external", "external": {"url": cover_url}}} ) return r.status_code == 200 def archive_page(page_id): r = requests.patch( f"https://api.notion.com/v1/pages/{page_id}", headers=HEADERS, json={"archived": True} ) return r.status_code == 200 def get_page(page_id): r = requests.get(f"https://api.notion.com/v1/pages/{page_id}", headers=HEADERS) if r.status_code == 200: return r.json() print(f" ERROR getting page {page_id}: {r.status_code} {r.text[:200]}") return None def screenshot_url(url, output_path): """Take screenshot with playwright, convert to jpg with imagemagick""" png_path = output_path.replace('.jpg', '.png') try: result = subprocess.run( ["playwright", "screenshot", "--viewport-size", "1400,900", "--wait-for-timeout", "5000", url, png_path], capture_output=True, text=True, timeout=30 ) if result.returncode != 0: print(f" Playwright error: {result.stderr[:200]}") return False # Convert to jpg, resize to max 1200px wide subprocess.run( ["convert", png_path, "-resize", "1200x>", "-quality", "85", output_path], capture_output=True, timeout=15 ) if os.path.exists(png_path): os.remove(png_path) return os.path.exists(output_path) and os.path.getsize(output_path) > 5000 except subprocess.TimeoutExpired: print(f" Timeout for {url}") return False except Exception as e: print(f" Exception: {e}") return False def is_ce_taste_dna_url(url): return "curiousendeavor.com/visual-research/taste-dna/" in url def is_screenshotone_url(url): return "screenshotone.com" in url def process_entry(page_id): """Process a single entry. Returns (title, success, filename_or_reason)""" page = get_page(page_id) if not page: return ("unknown", False, "could not fetch page") title = get_title(page) link = get_link(page) slug = slugify(title) or hashlib.md5(page_id.encode()).hexdigest()[:12] filename = f"{slug}.jpg" output_path = os.path.join(COVER_DIR, filename) print(f"Processing: {title}") print(f" Link: {link}") if not link: return (title, False, "no link URL") # Decode HTML entities link = decode_html_entities(link) # Handle CE taste-dna URLs if is_ce_taste_dna_url(link): # Extract filename from URL url_path = urllib.parse.urlparse(link).path local_path = "/root/.openclaw/workspace/public" + url_path # Try with and without trailing slash candidates = [local_path, local_path.rstrip('/')] # Also try common image extensions for ext in ['', '.jpg', '.jpeg', '.png', '.webp']: candidates.append(local_path.rstrip('/') + ext) found = False for candidate in candidates: if os.path.isfile(candidate): subprocess.run(["convert", candidate, "-resize", "1200x>", "-quality", "85", output_path], capture_output=True) found = True break elif os.path.isdir(candidate): # Check for index or image files inside for f in os.listdir(candidate): if f.lower().endswith(('.jpg', '.jpeg', '.png', '.webp')): subprocess.run(["convert", os.path.join(candidate, f), "-resize", "1200x>", "-quality", "85", output_path], capture_output=True) found = True break if found: break if found and os.path.exists(output_path) and os.path.getsize(output_path) > 1000: return (title, True, filename) return (title, False, "CE taste-dna file not found locally") # Handle ScreenshotOne URLs if is_screenshotone_url(link): target_url = extract_screenshotone_url(link) if target_url: link = target_url print(f" Extracted from ScreenshotOne: {link}") else: return (title, False, "could not extract URL from ScreenshotOne") # Screenshot the URL success = screenshot_url(link, output_path) if not success: # Try alternate approaches for LinkedIn if "linkedin.com" in link: alt_url = link + ("&" if "?" in link else "?") + "trk=public_post" success = screenshot_url(alt_url, output_path) if success: return (title, True, filename) return (title, False, "screenshot failed or too small") def main(): print(f"=== Step 1: Unarchiving {len(PAGE_IDS)} entries ===") for i, pid in enumerate(PAGE_IDS): pid = pid.strip() ok = unarchive_page(pid) if (i+1) % 10 == 0: print(f" Unarchived {i+1}/{len(PAGE_IDS)}") if not ok: print(f" WARNING: Failed to unarchive {pid}") time.sleep(0.35) # Rate limit print("Unarchiving complete.") print(f"\n=== Steps 2-3: Fetching pages and capturing screenshots ===") results = [] # (page_id, title, success, detail) for i, pid in enumerate(PAGE_IDS): pid = pid.strip() title, success, detail = process_entry(pid) results.append((pid, title, success, detail)) print(f" [{i+1}/{len(PAGE_IDS)}] {'✓' if success else '✗'} {title} -> {detail}") time.sleep(0.35) # Save results for later steps with open("/root/.openclaw/workspace/recover_results.json", "w") as f: json.dump(results, f, indent=2) successes = [(pid, t, d) for pid, t, s, d in results if s] failures = [(pid, t, d) for pid, t, s, d in results if not s] print(f"\n=== Results: {len(successes)} succeeded, {len(failures)} failed ===") print(f"\nSuccesses:") for pid, t, d in successes: print(f" ✓ {t} -> {d}") print(f"\nFailures:") for pid, t, d in failures: print(f" ✗ {t}: {d}") # Step 4: Deploy if successes: print("\n=== Step 4: Deploying to Vercel ===") os.chdir("/root/.openclaw/workspace/public") subprocess.run(["git", "add", "-A"], capture_output=True) subprocess.run(["git", "commit", "-m", f"recovered: {len(successes)} taste entries with fresh Playwright captures"], capture_output=True) result = subprocess.run(["git", "push", "origin", "main"], capture_output=True, text=True) print(f" Push: {result.returncode}") print(" Waiting 25s for Vercel deploy...") time.sleep(25) # Set covers print("\n=== Setting Notion covers ===") for pid, title, filename in successes: cover_url = f"{BASE_URL}/{filename}" ok = set_cover(pid, cover_url) print(f" {'✓' if ok else '✗'} Cover set for {title}") time.sleep(0.35) # Step 5: Re-archive failures if failures: print(f"\n=== Step 5: Re-archiving {len(failures)} failures ===") for pid, title, reason in failures: ok = archive_page(pid) print(f" {'✓' if ok else '✗'} Re-archived: {title} ({reason})") time.sleep(0.35) print(f"\n=== FINAL REPORT ===") print(f"Recovered: {len(successes)}") print(f"Re-archived: {len(failures)}") if __name__ == "__main__": main()