"""Profile discovery, protocol v2 (recorded). For each product and site: step 1 query "<product> <site> reviews"
(Trustpilot: "<product> trustpilot"); step 2, only if step 1 yields no valid profile, query "<product> site:<domain>".
A result is a valid profile only if its URL matches the site's product-profile pattern AND (for G2, Capterra, GetApp,
Software Advice) the normalised product name appears in the URL slug or title, or (for Trustpilot) the reviewed
domain equals the vendor's own website domain from the roster. Anything else is "not found under protocol".
Usage: python3 scripts/discover.py cohort1 2026-09-02"""
import json,urllib.request,base64,os,hashlib,re,time,sys,glob
cohort,date=sys.argv[1],sys.argv[2]
login,pw=os.environ['DATAFORSEO_LOGIN'],os.environ['DATAFORSEO_PASSWORD']
auth='Basic '+base64.b64encode(f'{login}:{pw}'.encode()).decode()
import sys as _s; _s.path.insert(0,'scripts'); from roster import COHORTS
products=[(n,d,t) for n,d,t,_ in COHORTS[cohort]]
sites=[('G2','g2.com',re.compile(r'^https?://(www\.)?g2\.com/products/[^/]+/(reviews|pricing)?/?$')),('Capterra','capterra.com',re.compile(r'^https?://(www\.)?capterra\.com/p/\d+/[^/]+/(reviews/?)?$')),('GetApp','getapp.com',re.compile(r'^https?://(www\.)?getapp\.com/[^/]+/a/[^/]+/(reviews/?)?$')),('Software Advice','softwareadvice.com',re.compile(r'^https?://(www\.)?softwareadvice\.com/[^/]+/[^/]+-profile/(reviews/?)?$')),('Trustpilot','trustpilot.com',re.compile(r'^https?://(www\.)?trustpilot\.com/review/([^/?#]+)/?$'))]
def serp(kw):
    body=[{"keyword":kw,"location_code":2840,"language_code":"en","depth":10}]
    req=urllib.request.Request('https://api.dataforseo.com/v3/serp/google/organic/live/advanced',data=json.dumps(body).encode(),headers={'Authorization':auth,'Content-Type':'application/json'})
    raw=urllib.request.urlopen(req,timeout=120).read(); h=hashlib.md5(raw).hexdigest()[:8]; open(f'raw_responses/dataforseo/serp-{re.sub(r"[^a-z0-9]+","-",kw.lower())[:50]}-{h}.json','wb').write(raw)
    d=json.loads(raw); return [it for t in d['tasks'] for r in (t.get('result') or []) for it in (r.get('items') or []) if it.get('type')=='organic']
def norm(s): return re.sub(r'[^a-z0-9]+','',s.lower())
def valid(site,pat,url,title,tokens,vdomain):
    m=pat.match(url or '')
    if not m: return False
    if site=='Trustpilot':
        d=m.group(2).lower().replace('www.',''); return d==vdomain
    hay=norm(url)+' '+norm(title or '')
    return any(norm(t) in hay for t in tokens)
rows=[]
for name,vdomain,tokens in products:
    for site,dom,pat in sites:
        q1=f'{name} {site} reviews' if site!='Trustpilot' else f'{name} trustpilot'
        items=serp(q1); hit=next((it for it in items if valid(site,pat,it.get('url'),it.get('title'),tokens,vdomain)),None); step=1
        if not hit:
            q2=f'{name} site:{dom}'; items=serp(q2); hit=next((it for it in items if valid(site,pat,it.get('url'),it.get('title'),tokens,vdomain)),None); step=2
        rows.append({'product':name,'site':site,'protocol_step':step if hit else None,'query':q1 if step==1 else f'{name} site:{dom}','profile_url':hit['url'] if hit else '','title':(hit.get('title') or '')[:90] if hit else '','found':bool(hit)})
        print(f"{name:20} {site:16} step {step if hit else '-'} -> {hit['url'][:80] if hit else 'NOT FOUND under protocol'}",flush=True); time.sleep(0.3)
json.dump({'protocol':'v2: step 1 "<product> <site> reviews" (Trustpilot: "<product> trustpilot"); step 2 if none valid: "<product> site:<domain>"; valid = site product-profile URL pattern AND product token in URL slug or title (Trustpilot: reviewed domain equals the vendor website domain); DataForSEO Google organic, US, depth 10','observation_date':date,'rows':rows},open(f'data/profiles-{cohort}-{date}.json','w'),indent=1)
print('discovery v2 saved:',sum(1 for r in rows if r['found']),'of',len(rows),'found',flush=True)