"""Discovery trace (2 September 2026, after gate 2 r2): rebuilds, from the frozen DataForSEO responses only, every step 1 and
step 2 attempt for every product-site row: the query, every saved response file for that query (with SHA-256 and mtime; the
earlier of two files is the first discovery run of the day, the later is the recorded v2 run), the organic items in the file
used (the latest by mtime), the first item that satisfies the acceptance rule (site profile URL pattern and product token in the
URL slug or title; Trustpilot: reviewed domain equals the vendor domain), and whether that decision reproduces the row's recorded
profile_url. Writes data/discovery-trace-<cohort>-<date>.json. Usage: python3 scripts/trace.py cohort1 2026-09-02"""
import json,re,sys,os,glob,hashlib,datetime
sys.path.insert(0,'scripts'); from roster import COHORTS
cohort,date=sys.argv[1],sys.argv[2]
d=json.load(open(f'data/profiles-{cohort}-{date}.json')); rows=d['rows'] if isinstance(d,dict) else d
tokens={n:t for n,_,t,_ in COHORTS[cohort]}; vdomain={n:dm for n,dm,_,_ in COHORTS[cohort]}
SITES=[('G2','g2.com',re.compile(r'^https?://(www\.)?g2\.com/products/[^/]+/(reviews|pricing)?/?$')),('Capterra','capterra.com',re.compile(r'^https?://(www\.)?capterra\.com/p/\d+/[^/]+/(reviews/?)?$')),('GetApp','getapp.com',re.compile(r'^https?://(www\.)?getapp\.com/[^/]+/a/[^/]+/(reviews/?)?$')),('Software Advice','softwareadvice.com',re.compile(r'^https?://(www\.)?softwareadvice\.com/[^/]+/[^/]+-profile/(reviews/?)?$')),('Trustpilot','trustpilot.com',re.compile(r'^https?://(www\.)?trustpilot\.com/review/([^/?#]+)/?$'))]
pat={s:p for s,_,p in SITES}; sdom={s:dm for s,dm,_ in SITES}
def slugq(q): return re.sub(r'[^a-z0-9]+','-',q.lower()).strip('-')
def files_for(q):
    fs=sorted(glob.glob(f'raw_responses/dataforseo/serp-{slugq(q)}-*.json'),key=os.path.getmtime)
    return [{'file':f,'sha256':hashlib.sha256(open(f,'rb').read()).hexdigest(),'mtime_utc':datetime.datetime.fromtimestamp(os.path.getmtime(f),datetime.UTC).strftime('%Y-%m-%dT%H:%M:%SZ')} for f in fs]
def items(f):
    try: dd=json.load(open(f)); return [it for t in dd['tasks'] for r in (t.get('result') or []) for it in (r.get('items') or []) if it.get('type')=='organic']
    except Exception: return []
def valid(p,site,it):
    u=(it.get('url') or '').split('?')[0]; title=(it.get('title') or '').lower()
    if not pat[site].match(u): return False
    if site=='Trustpilot':
        m=pat[site].match(u); rd=m.group(2).lower().replace('www.',''); return rd==vdomain[p]
    slug=u.lower()
    return any(t in slug or t in title for t in tokens[p])
out=[]
for r in rows:
    p,site=r['product'],r['site']
    q1=f"{p} trustpilot" if site=='Trustpilot' else f"{p} {site} reviews"; q2=f"{p} site:{sdom[site]}"
    steps=[]; decision=None
    for k,q in ((1,q1),(2,q2)):
        fs=files_for(q); used=fs[-1] if fs else None; its=items(used['file']) if used else []
        hit=next((it for it in its if valid(p,site,it)),None)
        steps.append({'step':k,'query':q,'files':fs,'file_used':used['file'] if used else None,'file_used_sha256':used['sha256'] if used else None,'organic_items':len(its),'first_valid_url':(hit.get('url') if hit else None),'decision':('accepted' if hit else ('no valid result' if used else 'no saved response'))})
        if hit and decision is None: decision=hit.get('url')
        if hit: break
    rec={'product':p,'site':site,'steps':steps,'reproduced_profile_url':decision,'recorded_profile_url':r.get('profile_url') or None,'reproduces_record':((decision or None)==(r.get('profile_url') or None))}
    out.append(rec)
json.dump(out,open(f'data/discovery-trace-{cohort}-{date}.json','w'),indent=1)
ok=sum(1 for o in out if o['reproduces_record']); print(f'{cohort}: {ok} of {len(out)} rows reproduce the recorded discovery decision from the frozen SERP files')
for o in out:
    if not o['reproduces_record']: print('  mismatch:',o['product'],o['site'],'reproduced',o['reproduced_profile_url'],'recorded',o['recorded_profile_url'])