import re,json,csv,collections,statistics as st,pathlib,sys
H=pathlib.Path(sys.argv[1]); OUT=sys.argv[2]; rows=[]
def field(seg,key):
    m=re.search(r'"%s":(?:"([^"]*)"|(null|true|false|-?\d+(?:\.\d+)?))'%re.escape(key),seg)
    return None if not m else (m.group(1) if m.group(1) is not None else m.group(2))
def reward(seg,key):
    i=seg.find('"%s":{'%key)
    if i<0: return None
    j=seg.find('}',i); r=seg[i:j+1]
    return {k:field(r,k) for k in ('event','type','amountInCents','amountInPercentage','maxDuration')}
for f in sorted(H.glob('*.html')):
    s=f.read_text(encoding='utf-8',errors='ignore').replace('\\"','"')
    i=s.find('"holdingPeriodDays"')
    if i<0: rows.append({'slug':f.stem,'parsed':False}); continue
    seg=s[max(0,i-20000):i+6000]
    title=re.search(r'<title>([^<]*)</title>',s); title=title.group(1) if title else ''
    after=s[i:i+6000]
    r={'slug':f.stem,'parsed':True,'program':title.replace(' Affiliate Program | Dub','').replace(' | Dub',''),
       'domain':field(seg,'domain'),'holdingPeriodDays':field(after,'holdingPeriodDays'),
       'programCreatedAt':(field(after,'createdAt') or '').replace('$D','')[:10],
       'saleReward':reward(after,'saleReward'),'leadReward':reward(after,'leadReward'),'clickReward':reward(after,'clickReward'),
       'discount':('"discount":{' in after),'bounties':bool(re.search(r'"bounties":\[\{',after)),'autoApprove':field(after,'autoApprovePartnersEnabledAt') not in (None,'null')}
    rows.append(r)
ok=[r for r in rows if r['parsed']]; print('parsed',len(ok),'of',len(rows))
def n(x):
    try: return float(x)
    except: return None
out=[]
for r in ok:
    sr=r['saleReward'] or {}; lr=r['leadReward'] or {}; cr=r['clickReward'] or {}
    dur=sr.get('maxDuration'); durlabel={'0':'one-time','null':'lifetime',None:''}.get(dur, f'{dur} months' if dur else '')
    out.append({'program':r['program'],'domain':r['domain'],'page':'https://partners.dub.co/'+r['slug'],
      'sale_reward_type':sr.get('type') or '','sale_pct':sr.get('amountInPercentage') if sr.get('type')=='percentage' else '',
      'sale_flat_usd':(n(sr.get('amountInCents')) or 0)/100 if sr.get('type')=='flat' else '','sale_duration':durlabel,
      'lead_reward_usd':(n(lr.get('amountInCents')) or 0)/100 if lr else '','click_reward_usd':(n(cr.get('amountInCents')) or 0)/100 if cr else '',
      'holding_period_days':r['holdingPeriodDays'],'customer_discount':r['discount'],'bounties':r['bounties'],'auto_approve':r['autoApprove'],'program_created':r['programCreatedAt']})
with open(OUT,'w',newline='') as f:
    w=csv.DictWriter(f,fieldnames=list(out[0].keys())); w.writeheader(); w.writerows(out)
pct=[float(o['sale_pct']) for o in out if o['sale_pct'] not in ('',None,'null')]
flat=[o['sale_flat_usd'] for o in out if o['sale_flat_usd']!='']
sale=[o for o in out if o['sale_reward_type']]
print('sale rewards',len(sale),'pct',len(pct),'flat',len(flat),'lead',sum(1 for o in out if o['lead_reward_usd']!=''),'click',sum(1 for o in out if o['click_reward_usd']!=''),'none',sum(1 for o in out if not o['sale_reward_type'] and o['lead_reward_usd']=='' and o['click_reward_usd']==''))
print('pct median',st.median(pct),'mean',round(st.mean(pct),1)); c=collections.Counter(int(p) for p in pct); print('pct buckets',sorted(c.items()))
print('pct >=20:',sum(1 for p in pct if p>=20),'>=30:',sum(1 for p in pct if p>=30),'>=50:',sum(1 for p in pct if p>=50),'<=10:',sum(1 for p in pct if p<=10),'20-30 incl:',sum(1 for p in pct if 20<=p<=30))
print('flat median',st.median(flat),'mean',round(st.mean(flat),1),'buckets',sorted(collections.Counter(int(x) for x in flat).items()))
d=collections.Counter(o['sale_duration'] for o in sale); print('duration',d.most_common())
h=collections.Counter(o['holding_period_days'] for o in out); print('holding',h.most_common())
print('discount',sum(o['customer_discount'] for o in out),'bounties',sum(o['bounties'] for o in out),'auto',sum(o['auto_approve'] for o in out))
print('lead usd',sorted(o['lead_reward_usd'] for o in out if o['lead_reward_usd']!=''))
print('click usd',sorted(o['click_reward_usd'] for o in out if o['click_reward_usd']!=''))
m=collections.Counter(o['program_created'][:7] for o in out); print('created by month',sorted(m.items()))
top=sorted([o for o in out if o['sale_pct'] not in ('',None)],key=lambda o:-float(o['sale_pct']))[:6]; print([(o['program'],o['sale_pct'],o['sale_duration']) for o in top])
for lab in ('lifetime','12 months','one-time','24 months','6 months','3 months'):
    xs=[float(o['sale_pct']) for o in sale if o['sale_duration']==lab and o['sale_pct'] not in ('',None)]
    if xs: print(lab,'n',len(xs),'median pct',st.median(xs))
zeros=[o['program'] for o in out if o['sale_pct'] in ('0',) or o['sale_flat_usd']==0]; print('zero rewards',zeros)