Ë
    ÿ�j.D  ã            
       ó¾  — d Z ddlZddlZddlmZ ddlmZmZ ddlmZ ddl	m
Z
 ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZmZ ddlmZmZ ddlmZm Z m!Z!m"Z"m#Z# ddl$m%Z%m&Z&m'Z'm(Z( ddl)m*Z*  ejV                  e,«      Z-dZ.de/de0fd„Z1de2de2de2de3fd„Z4de2de2de2de0fd„Z5de2de3fd„Z6de2de7de2de2de8f
d„Z9de3de2ddfd„Z:d e3d!e3fd"„Z;d%de2d#e8de8fd$„Z<y)&zýFetch, normalise, persist and report one source's catalog.

Every write and delete is scoped by (source_kind, source_ref). The crawler, a
Shopify store and an HTTP API share one table, so an unscoped delete here would
erase another producer's products.
é    N)ÚCounter)ÚdatetimeÚtimezone)Úsql)ÚJson)Úcrawl)Úhttp)Úshopify)ÚCrawlSourceError)ÚHttpSourceError)Úcontent_hashÚ	normaliseÚquality_scoreÚrecord_hash)Ú
BRANDFORGEÚcheck_quota)Úcount_new_productsÚget_db_connectionÚget_settingÚmigrate_products_tableÚupdate_setting)Ú
KIND_CRAWLÚget_credentialsÚmark_source_statusÚtouch_last_synced)ÚShopQueryErrorgš™™™™™©?ÚexÚreturnc                 ó,   — t        | «      }d|v xs d|v S )NzHTTP 401zHTTP 403)Ústr)r   Úmessages     ú8/var/www/html/strategist-ai/app/services/catalog/sync.pyÚ_is_auth_failurer#   %   s"   € ô �"‹g€GØ˜Ð Ò9 J°'Ð$9Ð9ó    Ú	tenant_idÚsource_kindÚ
source_refc                 ó  — |st        d«      ‚t        «       }	 |j                  «       5 }|j                  t	        j
                  d«      j                  t	        j                  | «      «      |||f«       |j                  }d d d «       |j                  «        |j                  «        S # 1 sw Y   Œ+xY w# t        $ r* |j                  «        t        j                  d| d¬«       ‚ w xY w# |j                  «        w xY w)Nz"delete_stale requires a source_refzwDELETE FROM {}.strategist_products WHERE source_kind = %s AND source_ref = %s AND (synced_at IS NULL OR synced_at < %s)z&Could not delete stale products for %sT©Úexc_info)Ú
ValueErrorr   ÚcursorÚexecuter   ÚSQLÚformatÚ
IdentifierÚrowcountÚcommitÚcloseÚ	ExceptionÚrollbackÚloggerÚerror)r%   r&   r'   Úrun_startedÚconnÚcurÚremoveds          r"   Údelete_staler<   -   sÞ   € áäÐ=Ó>Ð>äÓ€DðØ�[‰[‹]ð 	#˜cØ�K‰KœŸ™ð<ó÷ ‰f”S—^‘^ IÓ.Ó/Ø˜j¨+Ð6ô8ð —l‘lˆG÷	#ð 	�‰ŒØð 	�
‰
�÷!	#ð 	#ûô ò Ø�‰ŒÜ�‰Ð=¸yØ"ð 	ô 	$àð	ûð 	�
‰
�ús/   ™B6 ©AB*Â B6 Â*B3Â/B6 Â63C)Ã)C, Ã,C>c                 óþ  — t        «       }	 |j                  «       5 }|j                  t        j                  d«      j                  t        j                  | «      «      ||f«       |j                  «       d   cddd«       |j                  «        S # 1 sw Y   nxY wn9# t        $ r- t        j                  d| ||d¬«       Y |j                  «        yw xY w	 |j                  «        y# |j                  «        w xY w)ah  Whether this source currently has any product rows -- used to stop
    the crawl throttle from skipping a source whose data was wiped out from
    under it (e.g. a manual cleanup) without also resetting last_synced_at.
    A throttle that only looks at the timestamp can otherwise report a
    source as "recently synced" while its catalogue is actually empty.z_SELECT EXISTS (SELECT 1 FROM {}.strategist_products WHERE source_kind = %s AND source_ref = %s)r   Nz-Could not check product presence for %s:%s:%sTr)   F)r   r,   r-   r   r.   r/   r0   Úfetchoner3   r4   r6   r7   )r%   r&   r'   r9   r:   s        r"   Úhas_any_productsr?   H   sæ   € ô Ó€DðØ�[‰[‹]ð 	%˜cØ�K‰KœŸ™ð>ó÷ ‰f”S—^‘^ IÓ.Ó/°+¸zÐ1JôLð —<‘<“> !Ñ$÷	%ð 	%ð 	�
‰
�÷	%ð 	%úð 	%øô ò Ü�‰ÐDØ ¨ZÀ$ð 	ô 	Hð
 à�
‰
�ðúð	%ð 	�
‰
�øˆ�
‰
�úsA   ŒB œABÁ8	B ÂBÂB ÂC* Â#CÃC* ÃCÃC* Ã*C<c                 ó  — t        «       }	 |j                  «       5 }|j                  t        j                  d«      j                  t        j                  | «      «      t        f«       |j                  }ddd«       |j                  «        rt        j                  d|| «       ||j                  «        S # 1 sw Y   ŒDxY w# t        $ r* |j                  «        t        j                  d| d¬«       ‚ w xY w# |j                  «        w xY w)a  Retire crawled products written before the website became a real source.

    Those rows carry source_kind 'crawl' with a NULL source_ref, so delete_stale
    -- which matches on source_ref -- can never see them. Left alone they would
    sit beside the properly-synced rows forever, duplicating every product under
    an old key and with no price, category or content hash.

    Scoped to NULL refs only: a row belonging to a registered website source has
    a ref and is left to the ordinary stale-delete.
    zPDELETE FROM {}.strategist_products WHERE source_kind = %s AND source_ref IS NULLNz+Retired %d unregistered crawl row(s) for %sz/Could not retire unregistered crawl rows for %sTr)   )r   r,   r-   r   r.   r/   r0   r   r1   r2   r6   Úinfor3   r4   r5   r7   )r%   r9   r:   r;   s       r"   Úreap_unregistered_crawl_rowsrB   a   sä   € ô Ó€DðØ�[‰[‹]ð 	#˜cØ�K‰KœŸ™ð@ó÷ ‰f”S—^‘^ IÓ.Ó/´*°ô@ð —l‘lˆG÷	#ð 	�‰ŒÙÜ�K‰KÐEØ ô,àð 	�
‰
�÷#	#ð 	#ûô ò Ø�‰ŒÜ�‰ÐFØ¨ð 	ô 	/àð	ûð 	�
‰
�ús/   ŒC œAB8Á52C Â8CÂ=C Ã3C7Ã7C: Ã:DÚproductsc                 ó¤  — d}t        «       }	 |j                  «       5 }|D �])  }|j                  t        j                  d«      j                  t        j                  | «      «      g |d   ‘|d   ‘|d   ‘|d   ‘|d   ‘|d   ‘|‘|‘|d	   ‘|d
   ‘|d   ‘|d   ‘|d   ‘|d   ‘|d   ‘|d   ‘|d   ‘|d   ‘|d   ‘|d   ‘t        |d   «      ‘t        |«      ‘t        |«      ‘t        |«      ‘t        |d   «      ‘|d   ‘|d   ‘|d   ‘|d   ‘|d   ‘|d   ‘t        |j                  d«      xs g «      ‘|‘­«       |dz  }�Œ, 	 d d d «       |j                  «        d|i|j                  «        S # 1 sw Y   Œ-xY w# t        $ r* |j                  «        t         j#                  d | d!¬"«       ‚ w xY w# |j                  «        w xY w)#Nr   aq  
                    INSERT INTO {}.strategist_products
                        (product_key, name, description, image_url, product_url,
                         category, source_kind, source_ref, external_id, brand,
                         taxonomy_path, taxonomy_source, raw_category,
                         price_cents, price_max_cents, compare_at_cents, currency,
                         on_sale, in_stock, status, attributes, quality_score,
                         content_hash, record_hash, tenant_relations, rating,
                         review_count, featured_rank, missing_fields,
                         price_reference_cents, fx_rate_used, ctas, synced_at,
                         extracted_at)
                    VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,
                            %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s, CURRENT_TIMESTAMP)
                    ON CONFLICT (product_key) DO UPDATE SET
                        name = EXCLUDED.name,
                        description = EXCLUDED.description,
                        image_url = EXCLUDED.image_url,
                        product_url = EXCLUDED.product_url,
                        category = EXCLUDED.category,
                        source_kind = EXCLUDED.source_kind,
                        source_ref = EXCLUDED.source_ref,
                        external_id = EXCLUDED.external_id,
                        brand = EXCLUDED.brand,
                        taxonomy_path = EXCLUDED.taxonomy_path,
                        taxonomy_source = EXCLUDED.taxonomy_source,
                        raw_category = EXCLUDED.raw_category,
                        price_cents = EXCLUDED.price_cents,
                        price_max_cents = EXCLUDED.price_max_cents,
                        compare_at_cents = EXCLUDED.compare_at_cents,
                        currency = EXCLUDED.currency,
                        on_sale = EXCLUDED.on_sale,
                        in_stock = EXCLUDED.in_stock,
                        status = EXCLUDED.status,
                        attributes = EXCLUDED.attributes,
                        quality_score = EXCLUDED.quality_score,
                        content_hash = EXCLUDED.content_hash,
                        record_hash = EXCLUDED.record_hash,
                        tenant_relations = EXCLUDED.tenant_relations,
                        rating = EXCLUDED.rating,
                        review_count = EXCLUDED.review_count,
                        featured_rank = EXCLUDED.featured_rank,
                        missing_fields = EXCLUDED.missing_fields,
                        price_reference_cents = EXCLUDED.price_reference_cents,
                        fx_rate_used = EXCLUDED.fx_rate_used,
                        ctas = EXCLUDED.ctas,
                        synced_at = EXCLUDED.synced_at,
                        extracted_at = CURRENT_TIMESTAMP
                Úproduct_keyÚnameÚdescriptionÚ	image_urlÚproduct_urlÚcategoryÚexternal_idÚbrandÚtaxonomy_pathÚtaxonomy_sourceÚraw_categoryÚprice_centsÚprice_max_centsÚcompare_at_centsÚcurrencyÚon_saleÚin_stockÚstatusÚ
attributesÚtenant_relationsÚratingÚreview_countÚfeatured_rankÚmissing_fieldsÚprice_reference_centsÚfx_rate_usedÚctasé   Úsavedz!Could not persist products for %sTr)   )r   r,   r-   r   r.   r/   r0   r   r   r   r   Úgetr2   r3   r4   r5   r6   r7   )	r%   rC   r&   r'   r8   ra   r9   r:   Úps	            r"   Úpersistrd   ‚   sÊ  € à€EÜÓ€DðGØ�[‰[‹]ð >	˜cØó =�Ø—‘œCŸG™Gð /%ó /÷^ ‘VœCŸN™N¨9Ó5Ó6ð 9Ø�mÑ$ð9Ø&'¨¡ið9Ø12°=Ñ1Að9ØCDÀ[Á>ð9à�mÑ$ð9à&'¨
¡mð9à5@ð9àBLð9ð �mÑ$ð9ð '(¨¡jð9ð 34°OÑ2Dð9ð Ð'Ñ(ð	9ð +,¨NÑ*;ð	9ð >?¸}Ñ=Mð	9ð
 Ð'Ñ(ð9ð
 +,Ð,>Ñ*?ð9ð
 BCÀ:Áð9ð �i‘Lð9ð #$ J¡-ð9ð 23°8±ð9ô ˜˜<™Ó)ð9ô ,9¸Ó+;ð9ô ! “Oð9ô &1°£^ð9ô 6:¸!Ð<NÑ:OÓ5Pð9ð �h‘Kð9ð "# >Ñ!2ð9ð 56°oÑ4Fð9ð Ð&Ñ'ð9ð *+Ð+BÑ)Cð9ð �nÑ%ð9ô (,¨A¯E©E°&«MÒ,?¸RÓ'@ð9ð CNñ9ô_;ðx ˜‘
’ñ{=÷>	ð~ 	�‰ŒØ˜Ðð 	�
‰
�÷M>	ð >	ûôB ò Ø�‰ŒÜ�‰Ð8¸)ÈdˆÔSØðûð
 	�
‰
�ús/   ŽF žD0E;ÅF Å;FÆ F Æ3F:Æ:F= Æ=GÚpercentÚstepc                 ó|   — | €y	  | t        |«      |«       y# t        $ r t        j                  dd¬«       Y yw xY w)ztProgress is best-effort. The job is spending real money on model calls;
    a reporting failure must never abort it.NzProgress callback failedTr)   )Úintr4   r6   Úwarning)Úprogressre   rf   s      r"   Ú_reportrk   Ð   sC   € ð ÐØðBÙ”�W“˜tÕ$øÜò BÜ�‰Ð1¸DˆÖAðBús   … ˜ ;º;ÚstartÚendc                 ó   ‡ ‡‡— ‰ €yˆˆ ˆfd„S )z:Maps a sub-call's own 0-100 into this stage's slice of it.Nc                 óH   •— t        ‰‰‰‰z
  t        | «      z  dz  z   |«      S )Néd   )rk   rh   )re   rf   rm   rj   rl   s     €€€r"   ú<lambda>z_scaled.<locals>.<lambda>ß   s)   ø€ ¤¨°5¸CÀ%¹KÌ3ÈwË<Ñ;WÐ[^Ñ;^Ñ3^Ð`dÓ!e€ r$   © )rj   rl   rm   s   ```r"   Ú_scaledrs   Û   s   ú€ àÐØÝeÐer$   Úsourcec              ƒ   óx  ‡‡K  — t        j                  t        j                  «      }|d   |d   |d   }}}t	        ||«      }|s|t
        k7  rt        d|› d|› �«      ‚t        |dd|› d�«       	 |d	k(  r�t        j                  ||d
   «      ƒ d {  –—† }|D �	cg c]  }	t        j                  |	|«      ‘Œ }
}	|j                  d«      |j                  d«      d |j                  d«      xs |j                  d«      g dœŠnì|dk(  r}t        j                  |||«      ƒ d {  –—† }|j                  d«      }|D �cg c]  }t        j                  |||d¬«      ‘Œ }
}|j                  d«      d |j                  d«      |g dœŠnj|t
        k(  rSt        j                  ||| t        |dd«      ¬«      ƒ d {  –—† }
|j                  d«      d d |j                  d«      g dœŠnt        d|› �«      ‚t)        | d«      Š‰s‰d   Št+        | dˆfd„«       ‰‰d<   ˆfd„|
D «       D �	cg c]  }	|	�|	‘Œ }}	t-        |
«      t-        |«      }}||z
  }|rt/        ||z  d«      nd}t1        d„ |D «       «      }t        |dd|› d�«       t3        | dt4        t7        | |D �cg c]  }|d   ‘Œ	 c}«      ¬ «       t9        | «       t;        | ||||«       t=        | |||«      }|t
        k(  r|t?        | «      z  }t        |d!d|› d�«       |D �cg c]  }tA        |«      ‘Œ }}|tB        kD  rtD        jG                  d"|d!z  ||«       |rAtI        d#„ |D «       «      jK                  d$«      d%   d%   }tD        jG                  d&|||||«       	 tM        |||«       ||||||||r!t/        tS        jT                  |«      d*«      d+œS d d+œS 7 �Œ)c c}	w 7 �Œ¡c c}w 7 �Œ# t        t         t"        f$ r}t%        |«      rt'        ||d«       ‚ d }~ww xY wc c}	w c c}w c c}w # tN        $ r tD        jQ                  d'||d(¬)«       Y Œ©w xY w­w),NÚkindÚexternal_refÚconfigzno credentials stored for ú:é
   zsyncing z	 (1 of 1)r
   Úaccess_tokenÚ	shop_nameÚprimary_domainÚcurrency_coderS   )r|   r}   Úurl_templaterS   Úknown_brandsÚhttp_apiF)Úrequire_currencyr   é<   )rj   zunsupported source kind: r7   Úreference_currencyc                 ó   •— ‰S ©Nrr   )Ú_r„   s    €r"   rq   zsync_source.<locals>.<lambda>2  s   ø€ ÐBT€ r$   c              3   ó6   •K  — | ]  }t        |‰«      –— Œ y ­wr†   )r   )Ú.0ÚspÚcontexts     €r"   ú	<genexpr>zsync_source.<locals>.<genexpr>6  s   øè ø€ ÒD¨b”9˜R ×)ÑDùs   ƒé   g        c              3   ó,   K  — | ]  }|d    sŒ	d–— Œ y­w)r\   r`   Nrr   )r‰   rc   s     r"   rŒ   zsync_source.<locals>.<genexpr><  s   è ø€ ÒB˜1¨aÐ0@Ó.A”QÑBùs   ‚
�Úproduct_countrE   )Úproduct_nameÚrequested_amountrp   zWRejection rate %.1f%% for %s:%s suggests a normaliser bug rather than bad merchant datac              3   ó4   K  — | ]  }|d    D ]  }|–— Œ Œ y­w)r\   Nrr   )r‰   rc   Úfs      r"   rŒ   zsync_source.<locals>.<genexpr>`  s1   è ø€ ò @Ø¨1Ð-=Ñ+>ò@Ø&'ŒAð@Øñ@ùs   ‚r`   r   zK%d of %d products for %s:%s are incomplete; most common missing field is %rz)Could not record last_synced_at for %s:%sTr)   é   )ÚfetchedÚ
normalisedÚrejectedÚrejection_rateÚ
incompletera   ÚdeletedÚquality_median)+r   Únowr   Úutcr   r   r+   rk   Úcatalog_shopifyÚfetch_productsÚto_source_productrb   Úcatalog_httpÚcatalog_crawlrs   r   r   r   r#   r   r   r   ÚlenÚroundÚsumr   r   r   r   rd   r<   rB   r   ÚREJECTION_ALERT_RATIOr6   ri   r   Úmost_commonr   r4   r7   Ú
statisticsÚmedian)r%   rt   rj   r8   rv   Úrefrx   ÚcredentialsÚnodesÚnÚsource_productsÚrecordsrS   Úrr   r–   r•   Úkeptr—   Úrater™   rc   r;   Úscoresr§   r‹   r„   s                            @@r"   Úsync_sourcer´   â   sÒ  ùè ø€ Ü—,‘,œxŸ|™|Ó,€KØ˜v™¨¨~Ñ(>ÀÀxÑ@Pˆvˆ#€Dô " $¨Ó,€KÙ˜4¤:Ò-ÜÐ5°d°V¸1¸S¸EÐBÓCÐCäˆH�b˜H S E¨Ð3Ô4ð;Ø�9ÒÜ)×8Ñ8Ø�[ Ñ0ó2÷ 2ˆEð ).ö/Ø#$ô  /×@Ñ@ÀÀCÕHð /ˆOð /ð $ŸZ™Z¨Ó4Ø"(§*¡*Ð-=Ó">Ø $Ø"ŸJ™J Ó7ÒQ¸6¿:¹:ÀjÓ;QØ "ñ‰Gð �ZÒÜ(×7Ñ7¸ÀÈSÓQ×QˆGð —z‘z *Ó-ˆHð !ö"ð ô ×.Ñ.Ø�v˜s°Uö<ð"ˆOð "ð
 $ŸZ™Z¨Ó4Ø"&Ø &§
¡
¨>Ó :Ø$Ø "ñ‰Gð ”ZÒÜ$1×$@Ñ$@Ø˜˜Y´¸À2ÀrÓ1Jô%L÷ LˆOð $ŸZ™Z¨Ó4Ø"&ð !%Ø"ŸJ™J zÓ2Ø "ñ‰Gô Ð8¸¸Ð?Ó@Ð@ô % YÐ0DÓEÐÙØ$ ZÑ0ÐÜ�yÐ"6Ó8TÔUØ$6€GÐ Ñ!ó E°OÔDö$˜à�]ò ð $€Jð $ô ˜Ó(¬#¨j«/ˆT€GØ˜‰~€HÙ+2Œ5�˜GÑ# QÔ'¸€DÜÑB 
ÔBÓB€JäˆH�b˜H S E¨Ð3Ô4ô �	˜?¼Ü!3Ø¸*ÖE°Q  -Ó 0ÒEó"GõHô ˜9Ô%ÜˆI�z 4¨¨kÔ:ô ˜9 d¨C°Ó=€GØŒzÒð 	Ô/°	Ó:Ñ:ˆäˆH�c˜X c U¨)Ð4Ô5à(2Ö3 1Œm˜AÕÐ3€FÐ3ØÔ#Ò#Ü�‰ð ;Ø<@À3¹JÈÈcô	Sáô ñ @Ø!ô@ó @ß@KÁÈAÃÈqñRØRSñUˆä�‰ð -Ø.8¸$ÀÀcØ"ô	$ð$ô 	˜$  [Ô1ð ØØØØ ØØÙAGœ%¤
× 1Ñ 1°&Ó 9¸1Ó=ñ	ð 	ð NRñ	ð 	ð}2ûò/ð Rûò"ðLúô œOÔ-=Ð>ò ô
 ˜BÔÜ˜t S¨'Ô2Øûðüò&$ùò(  Fùò 4øô$ ò $Ü�‰Ð@À$ÈØ"ð 	ö 	$ð$üsÅ   „A,P:Á1!O ÂN;ÂO ÂN>Â8A*O Ä"OÄ#O Ä< OÅAO Æ5OÆ69O Ç/5P:È$PÈ/A)P:ÊPÊ$A P:ÌPÌA)P:Î P Î.P:Î;O Î>O ÏO ÏP Ï"O;Ï;P Ð P:Ð"P7Ð4P:Ð6P7Ð7P:r†   )=Ú__doc__Úloggingr¨   Úcollectionsr   r   r   Úpsycopg2r   Úpsycopg2.extrasr   Úapp.services.catalogr   r¢   r	   r¡   r
   rž   Úapp.services.catalog.crawlr   Úapp.services.catalog.httpr   Úapp.services.catalog.normaliser   r   r   r   Úapp.services.infra.quotasr   r   Úapp.services.infra.databaser   r   r   r   r   Úapp.services.catalog.sourcesr   r   r   r   Ú'app.services.integrations.shopify_oauthr   Ú	getLoggerÚ__name__r6   r¦   r4   Úboolr#   r    rh   r<   r?   rB   ÚlistÚdictrd   rk   rs   r´   rr   r$   r"   ú<module>rÇ      sZ  ðñó Û Ý ß 'å Ý  å 7Ý 5Ý ;Ý 7Ý 5÷ó ÷ >÷õ ÷ó õ Cà	ˆ×	Ñ	˜8Ó	$€àÐ ð:˜ð : tó :ð˜Cð ¨cð ¸sð Ø!$óð6 ð °#ð À3ð È4ó ð2¨Cð °Có ðBK�sð K dð K¸ð KÈ#ð KØ óKð\B˜sð B¨#ð B°$ó Bðf˜Sð f só fñU ð U¨dð UÀdô Ur$   