"""Protocol step 3, second implementation (2 September 2026, after gate 2): a deterministic pattern probe for EVERY product-site
pair that steps 1 and 2 left unfound, on all five sites. Every attempt is saved as its own file with status, bytes, SHA-256,
UTC time, canonical URL (parsed order-independently), title and the acceptance decision. Acceptance: the canonical URL (or the
final URL when the page carries none) matches the site's profile pattern and is not a comparison page; the page title or JSON-LD
name carries a roster token; the page carries an aggregateRating block (Trustpilot: a businessUnit block). Rows already accepted
by the first probe run are kept as they are (their file is hashed, not refetched). Capterra has no deterministic profile pattern
(numeric id in the path) and is recorded as such. Usage: python3 scripts/probe2.py cohort1 2026-09-02"""
import json,os,re,sys,urllib.request,urllib.parse,datetime,hashlib,time,subprocess
sys.path.insert(0,'scripts'); from roster import COHORTS
cohort,date=sys.argv[1],sys.argv[2]
KEY=os.environ['SCRAPEDO_API_KEY']
d=json.load(open(f'data/profiles-{cohort}-{date}.json')); rows=d['rows'] if isinstance(d,dict) else d
old=[]
pf=f'data/profiles-probe-{cohort}-{date}.json'
if os.path.exists(pf): old=json.load(open(pf))
tokens={n:t for n,_,t,_ in COHORTS[cohort]}; variants={n:v for n,_,_,v in COHORTS[cohort]}; vdomain={n:dm for n,dm,_,_ in COHORTS[cohort]}
PAT={'G2':re.compile(r'^https?://(www\.)?g2\.com/products/[^/]+/(reviews|pricing)?/?$'),'Capterra':re.compile(r'^https?://(www\.)?capterra\.com/p/\d+/[^/]+/(reviews/?)?$'),'GetApp':re.compile(r'^https?://(www\.)?getapp\.com/[^/]+/a/[^/]+/(reviews/?)?$'),'Software Advice':re.compile(r'^https?://(www\.)?softwareadvice\.com/[^/]+/[^/]+-profile/(reviews/?)?$'),'Trustpilot':re.compile(r'^https?://(www\.)?trustpilot\.com/review/([^/?#]+)/?$')}
def fetch(u,site):
    if site=='Trustpilot':
        r=subprocess.run(['curl','-s','-m','60','-o','-','-w','\n%{http_code}','-A','Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128 Safari/537.36','-L',u],capture_output=True)
        out=r.stdout.decode('utf-8','ignore'); body,_,code=out.rpartition('\n'); 
        if code.strip()=='200' and len(body)>20000: return int(code),body,'curl'
    q="https://api.scrape.do/?token="+KEY+"&url="+urllib.parse.quote(u,safe='')+"&super=true&geoCode=us"
    try:
        r=urllib.request.urlopen(urllib.request.Request(q,headers={"User-Agent":"Mozilla/5.0"}),timeout=120); return r.status,r.read().decode('utf-8','ignore'),'scrape.do'
    except urllib.error.HTTPError as e: return e.code,'','scrape.do'
def exact_name_ok(name,label):
    # a JSON-LD name equal to the roster label qualifies in the sensitivity cohorts only (added for G2's bare 'Crystal'); never in cohort 1 or the appendix
    return cohort in ('cohort2','cohort3') and bool(name) and re.sub(r'\s+',' ',name.strip().lower())==re.sub(r'\s+',' ',label.strip().lower())
def canonical(h):
    for tag in re.findall(r'<link[^>]+>',h[:400000]):
        if re.search(r'rel=["\']canonical["\']',tag):
            m=re.search(r'href=["\']([^"\']+)["\']',tag)
            if m: return m.group(1)
    return None
def jsonld_name(h):
    for blk in re.findall(r'<script[^>]*type="application/ld\+json"[^>]*>(.*?)</script>',h,re.S):
        try: o=json.loads(blk)
        except Exception: continue
        st=[o]
        while st:
            x=st.pop()
            if isinstance(x,dict):
                if x.get('@type') in ('SoftwareApplication','Product') and x.get('name'): return str(x['name'])
                st+=[v for v in x.values() if isinstance(v,(dict,list))]
            elif isinstance(x,list): st+=x
    return None
if '--rehash' in sys.argv:
    # recompute hash and size from disk and backfill every per-attempt field (canonical, title, JSON-LD name, pattern and token
    # decisions) from the saved file, so that attempts kept from the first probe run carry the same fields as later ones
    for o in old:
        p=o['product']; site=o['site']
        for a in o.get('attempts',[]):
            if not (a.get('saved') and os.path.exists(a['saved'])): continue
            fb=open(a['saved'],'rb').read(); h=fb.decode('utf-8','ignore'); a['sha256']=hashlib.sha256(fb).hexdigest(); a['bytes']=len(fb)
            can=canonical(h); title=re.search(r'<title[^>]*>(.*?)</title>',h,re.S|re.I); title=title.group(1).strip() if title else None; name=jsonld_name(h)
            target=can or a.get('url') or ''
            # every derived field is recomputed from the saved bytes, never preserved from an earlier run
            a['canonical']=can; a['title']=(title or '')[:120]; a['jsonld_name']=name
            a['has_rating_block']=('"aggregateRating"' in h) or (site=='Trustpilot' and '"numberOfReviews"' in h)
            a['pattern_ok']=bool(PAT[site].match(target)) and '/compare/' not in target
            a['token_ok']=any(t in (title or '').lower() for t in tokens[p]) or any(t in (name or '').lower() for t in tokens[p]) or (site=='Trustpilot' and vdomain[p] in target) or exact_name_ok(name,p)
            a['accepted']=bool(a.get('status')==200 and a['pattern_ok'] and a['token_ok'] and '404' not in (title or ''))
        # the row's found state follows the recomputed attempts
        acc=[a for a in o.get('attempts',[]) if a.get('accepted')]
        if acc:
            a=acc[0]; o.update({'found':True,'url':a.get('url',o.get('url')),'canonical':a['canonical'],'saved':a['saved'],'sha256':a['sha256'],'bytes':a['bytes'],'status':a.get('status'),'title':a['title']})
        elif o.get('attempts'):
            for k in ('url','canonical','saved','sha256','bytes','status','title'): o.pop(k,None)
            o['found']=False
    json.dump(old,open(pf,'w'),indent=1); print('rehashed and backfilled',pf); sys.exit(0)
out=[]; n_acc=0
for r in rows:
    if r['found']: continue
    p=r['product']; site=r['site']
    prev=next((o for o in old if o['product']==p and o['site']==site and o.get('found')),None)
    if prev:
        fn=prev['saved']; h=open(fn,encoding='utf-8',errors='ignore').read(); fb=open(fn,'rb').read()
        prev['sha256']=hashlib.sha256(fb).hexdigest(); prev['bytes']=len(fb); prev['canonical']=canonical(h) or prev.get('canonical'); prev['protocol_step']='3 pattern probe'
        prev.setdefault('attempts',[{'url':prev['url'],'status':prev.get('status'),'saved':fn,'sha256':prev['sha256'],'fetched_utc':prev.get('fetched_utc'),'accepted':True,'note':'first probe run, kept'}])
        out.append(prev); n_acc+=1; continue
    slug=variants[p][0]
    if site=='G2': cands=[f"https://www.g2.com/products/{v}/reviews" for v in variants[p]]
    elif site=='GetApp': cands=[f"https://www.getapp.com/hr-employee-management-software/a/{v}/" for v in variants[p]]
    elif site=='Software Advice': cands=[f"https://www.softwareadvice.com/{c}/{v}-profile/" for v in variants[p] for c in ('hr','recruiting','pre-employment-testing')]
    elif site=='Trustpilot': cands=[f"https://www.trustpilot.com/review/{vdomain[p]}",f"https://www.trustpilot.com/review/www.{vdomain[p]}"]
    else: cands=[]
    rec={'product':p,'site':site,'protocol_step':'3 pattern probe','candidates':cands,'attempts':[],'found':False}
    if not cands: rec['note']='no deterministic profile pattern for this site (numeric id in the path)'; out.append(rec); continue
    for k,u in enumerate(cands,1):
        st,h,how=fetch(u,site); ts=datetime.datetime.now(datetime.UTC).strftime('%Y-%m-%dT%H:%M:%SZ')
        fn=f"raw/sites-{date}-probe/{slug}-{site.lower().replace(' ','-')}-attempt{k}.html"; os.makedirs(os.path.dirname(fn),exist_ok=True); open(fn,'w',encoding='utf-8').write(h)
        fb=open(fn,'rb').read()  # hash and size of the bytes actually saved
        can=canonical(h); title=re.search(r'<title[^>]*>(.*?)</title>',h,re.S|re.I); title=title.group(1).strip() if title else None
        name=jsonld_name(h); agg=('"aggregateRating"' in h) or (site=='Trustpilot' and '"numberOfReviews"' in h)
        target=can or u
        pat_ok=bool(PAT[site].match(target)) and '/compare/' not in target
        tok_ok=any(t in (title or '').lower() for t in tokens[p]) or any(t in (name or '').lower() for t in tokens[p]) or (site=='Trustpilot' and vdomain[p] in target) or exact_name_ok(name,p)  # exact roster label as the JSON-LD name also qualifies (added after the cohort 3 run: G2 lists Crystal under the bare name 'Crystal')
        acc=bool(st==200 and pat_ok and tok_ok and '404' not in (title or ''))  # a profile without an aggregateRating block (no rating displayed) is still a found profile; extract2 records it as not eligible
        a={'url':u,'status':st,'via':how,'saved':fn,'bytes':len(fb),'sha256':hashlib.sha256(fb).hexdigest(),'fetched_utc':ts,'canonical':can,'title':(title or '')[:120],'jsonld_name':name,'has_rating_block':agg,'pattern_ok':pat_ok,'token_ok':tok_ok,'accepted':acc}
        rec['attempts'].append(a); print(f"{p:20} {site:15} {st} attempt{k} accepted={acc} canonical={can} title={(title or '')[:60]}")
        if acc: rec.update({'found':True,'url':u,'canonical':can,'saved':fn,'sha256':a['sha256'],'fetched_utc':ts,'status':st,'title':a['title']}); n_acc+=1; break
        time.sleep(1)
    out.append(rec)
json.dump(out,open(pf,'w'),indent=1); print(f'saved {pf}: {n_acc} accepted of {len(out)} probed rows')