"""Supplementary capture (2 September 2026, after gate 2): the overall sub-ratings that the corroboration check needs are shown on
the Capterra and Software Advice PRODUCT pages; when discovery returned the /reviews/ page of a profile, this step fetches the
product page (the same profile URL without the trailing reviews/ segment), saves it with SHA-256 and UTC time, and records it in
data/supplement-<cohort>-<date>.json. It never changes counts or ratings, which stay on the captured page.
Usage: python3 scripts/supplement.py cohort1 2026-09-02"""
import json,os,re,sys,urllib.request,urllib.parse,datetime,hashlib
cohort,date=sys.argv[1],sys.argv[2]; KEY=os.environ['SCRAPEDO_API_KEY']
d=json.load(open(f'data/profiles-{cohort}-{date}.json')); rows=d['rows'] if isinstance(d,dict) else d
probe=json.load(open(f'data/profiles-probe-{cohort}-{date}.json')) if os.path.exists(f'data/profiles-probe-{cohort}-{date}.json') else []
def fetch(u):
    q="https://api.scrape.do/?token="+KEY+"&url="+urllib.parse.quote(u,safe='')+"&super=true&geoCode=us"
    r=urllib.request.urlopen(urllib.request.Request(q,headers={"User-Agent":"Mozilla/5.0"}),timeout=120); return r.status,r.read().decode('utf-8','ignore')
out=[]
for r in rows:
    if r['site'] not in ('Capterra','Software Advice'): continue
    url=r['profile_url'] if r['found'] else next((x.get('url') for x in probe if x['product']==r['product'] and x['site']==r['site'] and x.get('found')),None)
    if not url: continue
    slug=re.sub(r'[^a-z0-9]+','-',r['product'].lower()).strip('-')
    if re.search(r'/reviews/?$',url): pu=re.sub(r'/reviews/?$','/',url); kind='product page'
    elif r['site']=='Capterra': pu=url.rstrip('/')+'/reviews/'; kind='reviews page'   # Capterra's reviews-page header shows Ease of use and Customer Service
    else: continue
    fn=f"raw/sites-{date}-supplement/{slug}-{r['site'].lower().replace(' ','-')}-{kind.split()[0]}.html"; os.makedirs(os.path.dirname(fn),exist_ok=True)
    if os.path.exists(fn) and os.path.getsize(fn)>10000:
        h=open(fn,encoding='utf-8',errors='ignore').read(); st='cached'
        fetched=datetime.datetime.fromtimestamp(os.path.getmtime(fn),datetime.UTC).strftime('%Y-%m-%dT%H:%M:%SZ'); verified=datetime.datetime.now(datetime.UTC).strftime('%Y-%m-%dT%H:%M:%SZ')
    else:
        try: st,h=fetch(pu)
        except Exception as e: out.append({'product':r['product'],'site':r['site'],'supplement_page':pu,'kind':kind,'error':str(e)}); print(r['product'],r['site'],'ERR',e); continue
        open(fn,'w',encoding='utf-8').write(h); fetched=datetime.datetime.now(datetime.UTC).strftime('%Y-%m-%dT%H:%M:%SZ'); verified=None
    fb=open(fn,'rb').read()
    out.append({'product':r['product'],'site':r['site'],'captured_page':url,'supplement_page':pu,'kind':kind,'status':st,'saved':fn,'bytes':len(fb),'sha256':hashlib.sha256(fb).hexdigest(),'fetched_utc':fetched,'verified_utc':verified})
    print(f"{r['product']:20} {r['site']:15} {st} {kind:13} {pu}")
json.dump(out,open(f'data/supplement-{cohort}-{date}.json','w'),indent=1); print('saved',len(out),'supplement rows')