"""Capture step of the review cross-listing study. Reads data/profiles-<cohort>-<date>.json, fetches every found
profile (Scrape.do proxy for G2, Capterra, GetApp, Software Advice; direct for Trustpilot), saves the HTML under
raw/sites-<date>/<product>-<site>.html with a UTC timestamp, and extracts the displayed review count, rating and
five star bins with site-specific patterns. Bins are recorded as integers when the page shows counts and as
percentages when it shows only percentages (bins_kind). Anything not matched is left null and flagged for a manual
browser read; nothing is inferred. Run from the package directory: python3 scripts/capture.py cohort1 2026-09-02"""
import json,os,re,sys,html,time,datetime,urllib.parse,subprocess
cohort,date=sys.argv[1],sys.argv[2]
SD=os.environ['SCRAPEDO_API_KEY']
prof=json.load(open(f'data/profiles-{cohort}-{date}.json'))
out=[]
def text(h):
    h=re.sub(r'(?is)<(script|style|noscript|svg).*?</\1>',' ',h); return re.sub(r'\s+',' ',html.unescape(re.sub(r'<[^>]+>',' ',h)))
def fetch(url,site):
    if site=='Trustpilot':
        r=subprocess.run(['curl','-s','-m','60','-A','Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128 Safari/537.36','-L',url],capture_output=True); h=r.stdout.decode('utf-8','ignore')
        if len(h)>20000: return h,'direct'
    u=urllib.parse.quote(url,safe=''); r=subprocess.run(['curl','-s','-m','120',f'https://api.scrape.do/?token={SD}&url={u}'],capture_output=True); return r.stdout.decode('utf-8','ignore'),'scrape.do'
def ints(s): return int(s.replace(',',''))
def fetch_error(h):
    # added 2 September 2026 after the live gate: a proxy error JSON, an empty body or a non-HTML body is a capture failure
    s=h.strip()
    if not s: return 'empty body',None,None
    if s.startswith('{'):
        try:
            j=json.loads(s)
            if isinstance(j,dict) and 'StatusCode' in j and ('ErrorType' in j or 'ErrorCode' in j): return 'fetch-layer error response',int(j['StatusCode']),str(j.get('ErrorType') or j.get('ErrorCode'))
        except Exception: pass
    if not re.search(r'(?i)<html|<!doctype|<head|<body',s[:5000]): return 'non-HTML body',None,None
    return 'page',None,None
def extract(site,h):
    t=text(h); res={'count':None,'rating':None,'bins':None,'bins_kind':None,'evidence':[]}
    try:
        if site=='G2':
            m=re.search(r'(\d[\d,]*)\s+reviews?\b',t); 
            if m: res['count']=ints(m.group(1)); res['evidence'].append(m.group(0))
            m=re.search(r'(\d\.\d) out of 5',t) or re.search(r'"ratingValue":\s*"?(\d\.\d)',h)
            if m: res['rating']=float(m.group(1)); res['evidence'].append(m.group(0)[:40])
            bins=re.findall(r'(\d) stars?\D{0,40}?(\d[\d,]*)\s*(%?)',t)
            b={}; kind=None
            for star,val,pct in bins:
                if star in '12345' and star not in b: b[star]=ints(val); kind='percent' if pct else 'count'
            if len(b)==5: res['bins']=[b['5'],b['4'],b['3'],b['2'],b['1']]; res['bins_kind']=kind
        elif site in ('Capterra','GetApp','Software Advice'):
            m=re.search(r'(\d\.\d)\s*\(\s*(\d[\d,]*)\s*\)',t)
            if m: res['rating']=float(m.group(1)); res['count']=ints(m.group(2)); res['evidence'].append(m.group(0))
            else:
                m=re.search(r'(\d[\d,]*)\s+reviews?\b',t)
                if m: res['count']=ints(m.group(1)); res['evidence'].append(m.group(0))
            bins=re.findall(r'(\d) stars?\D{0,30}?(\d[\d,]*)\s*(%?)',t)
            b={}; kind=None
            for star,val,pct in bins:
                if star in '12345' and star not in b: b[star]=ints(val); kind='percent' if pct else 'count'
            if len(b)==5: res['bins']=[b['5'],b['4'],b['3'],b['2'],b['1']]; res['bins_kind']=kind
        elif site=='Trustpilot':
            m=re.search(r'(\d[\d,]*)\s+total\b',t) or re.search(r'"reviewCount":\s*"?(\d+)',h)
            if m: res['count']=ints(m.group(1)); res['evidence'].append(m.group(0)[:40])
            m=re.search(r'TrustScore\s*(\d\.\d)',t) or re.search(r'"ratingValue":\s*"?(\d\.\d)',h)
            if m: res['rating']=float(m.group(1)); res['evidence'].append(m.group(0)[:40])
            bins=re.findall(r'(\d)-star\D{0,30}?(\d[\d,]*)\s*(%?)',t)
            b={}; kind=None
            for star,val,pct in bins:
                if star in '12345' and star not in b: b[star]=ints(val); kind='percent' if pct else 'count'
            if len(b)==5: res['bins']=[b['5'],b['4'],b['3'],b['2'],b['1']]; res['bins_kind']=kind
    except Exception as e: res['evidence'].append('extract error '+str(e))
    return res
os.makedirs(f'raw/sites-{date}',exist_ok=True)
for r in prof['rows']:
    rec=dict(r)
    if not r['found']: rec.update({'status':'not found under protocol'}); out.append(rec); continue
    h,how=fetch(r['profile_url'],r['site']); ts=datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
    fn=f"raw/sites-{date}/{re.sub(r'[^a-z0-9]+','-',r['product'].lower())}-{re.sub(r'[^a-z0-9]+','-',r['site'].lower())}.html"
    open(fn,'w',encoding='utf-8').write(h)
    cs,hs,et=fetch_error(h)
    ex=extract(r['site'],h) if cs=='page' else {'count':None,'rating':None,'bins':None,'bins_kind':None,'evidence':[]}
    rec.update({'fetched_utc':ts,'fetch_method':how,'bytes':len(h),'saved':fn,'capture_status':cs,'http_status':hs,'error_type':et,**ex,
                'status':(('captured' if ex['count'] is not None else 'captured, needs manual read') if cs=='page' else f"profile found in discovery; capture unresolved ({hs if hs else cs})")})
    out.append(rec); print(f"{r['product']:20} {r['site']:16} {len(h):7}b count={ex['count']} rating={ex['rating']} bins={ex['bins']} ({ex['bins_kind']})")
    time.sleep(0.5)
json.dump({'protocol':prof['protocol'],'observation_date':date,'rows':out},open(f'data/capture-{cohort}-{date}.json','w'),indent=1)
print('saved data/capture-%s-%s.json'%(cohort,date))