+
    i/                     f   R t ^ RIHtHtHtHtHt ^RIHtH	t	 ]! R]	P                  ]	P                  ]	P                  ]	P                  ]	P                  ]	P                  ]	P                   ]	P"                  4	      tRR R lltR R ltR R	 ltR
 R ltR R ltR R ltR R ltR R ltR R ltR# )z2Normalization of raw API data to canonical schema.)AnyDictListTypeVarUnion)datesschemaTc                    V ^8  d   QhR\         \        ,          R\        R\        R\        R\         \        ,          /# )   items	from_dateto_daterequire_datereturn)r   r	   strbool)formats   "U/Users/bowang/.openclaw/workspace/skills/last30days-official/scripts/lib/normalize.py__annotate__r   
   sB     % %7%% % 	%
 
!W%    c                    . pV  Fc  pVP                   f   V'       g   VP                  V4       K,  VP                   V8  d   K?  VP                   V8  d   KR  VP                  V4       Ke  	  V# )a  Hard filter: Remove items outside the date range.

This is the safety net - even if the prompt lets old content through,
this filter will exclude it.

Args:
    items: List of items to filter
    from_date: Start date (YYYY-MM-DD) - exclude items before this
    to_date: End date (YYYY-MM-DD) - exclude items after this
    require_date: If True, also remove items with no date

Returns:
    Filtered list with only items in range (or unknown dates if not required)
)dateappend)r   r   r   r   resultitems   &&&&  r   filter_by_date_ranger   
   sd    ( F99d# 99y  99wd   Mr   c          	          V ^8  d   QhR\         \        \        \        3,          ,          R\        R\        R\         \        P
                  ,          /# r   r   r   r   r   )r   r   r   r   r   
RedditItem)r   s   "r   r   r   2   sH     9 9S#X99 9 
&

	9r   c                   . pV  EF  pRpVP                  R4      p\        V\        4      '       dG   \        P                  ! VP                  R4      VP                  R4      VP                  R4      R7      p. pVP                  R. 4       F|  pVP                  \        P                  ! VP                  R^ 4      VP                  R4      VP                  R	R
4      VP                  RR
4      VP                  RR
4      R7      4       K~  	  VP                  R4      p	\        P                  ! WV4      p
VP                  \        P                  ! VP                  RR
4      VP                  RR
4      VP                  RR
4      VP                  RR
4      V	V
VVVP                  R. 4      VP                  RR4      VP                  RR
4      R7      4       EK  	  V# )zNormalize raw Reddit items to schema.

Args:
    items: Raw Reddit items from API
    from_date: Start of date range
    to_date: End of date range

Returns:
    List of RedditItem objects
N
engagementscorenum_commentsupvote_ratio)r"   r#   r$   top_commentsr   author excerpturlr"   r   r&   r(   r)   idtitle	subredditcomment_insights	relevance      ?why_relevant)r+   r,   r)   r-   r   date_confidencer!   r%   r.   r/   r1   )
get
isinstancedictr   
Engagementr   Commentr   get_date_confidencer   )r   r   r   
normalizedr   r!   eng_rawr%   cdate_strr2   s   &&&        r   normalize_reddit_itemsr=   2   s    J
((<(gt$$**kk'*$[[8$[[8J ."-AeeGQ'UU6]uuXr*i,EE%$!  . 88F#33HQ&++xxb!((7B'#hh{B/+!%!XX&8"=hh{C0."5
 	5 P r   c          	          V ^8  d   QhR\         \        \        \        3,          ,          R\        R\        R\         \        P
                  ,          /# r   )r   r   r   r   r   XItem)r   s   "r   r   r   n   sF     - -S#X-- - 
&,,	-r   c                   . pV  EF9  pRpVP                  R4      p\        V\        4      '       dW   \        P                  ! VP                  R4      VP                  R4      VP                  R4      VP                  R4      R7      pVP                  R4      p\
        P                  ! WqV4      pVP                  \        P                  ! VP                  R	R
4      VP                  RR
4      VP                  RR
4      VP                  RR
4      VVVVP                  RR4      VP                  RR
4      R7	      4       EK<  	  V# )zNormalize raw X items to schema.

Args:
    items: Raw X items from API
    from_date: Start of date range
    to_date: End of date range

Returns:
    List of XItem objects
Nr!   likesrepostsrepliesquotes)rA   rB   rC   rD   r   r+   r'   textr)   author_handler/   r0   r1   )	r+   rE   r)   rF   r   r2   r!   r/   r1   )	r3   r4   r5   r   r6   r   r8   r   r?   )	r   r   r   r9   r   r!   r:   r<   r2   s	   &&&      r   normalize_x_itemsrG   n   s    J
((<(gt$$**kk'*I.I.{{8,	J 88F#33HQ&,,xxb!&"%#((?B7+!hh{C0."5

 
	! 8 r   c          	          V ^8  d   QhR\         \        \        \        3,          ,          R\        R\        R\         \        P
                  ,          /# r   )r   r   r   r   r   YouTubeItem)r   s   "r   r   r      sH     * *S#X** * 
&

	*r   c                B   . pV  EF  pVP                  R4      ;'       g    / p\        P                  ! VP                  R4      VP                  R4      VP                  R4      R7      pVP                  R4      pVP                  \        P                  ! VP                  RR4      VP                  R	R4      VP                  R
R4      VP                  RR4      VRVVP                  RR4      VP                  RR4      VP                  RR4      R7
      4       EK  	  V# )zNormalize raw YouTube items to schema.

Args:
    items: Raw YouTube items from yt-dlp
    from_date: Start of date range
    to_date: End of date range

Returns:
    List of YouTubeItem objects
r!   viewsrA   commentsrK   rA   r#   r   video_idr'   r,   r)   channel_namehightranscript_snippetr/   ffffff?r1   )
r+   r,   r)   rO   r   r2   r!   rQ   r/   r1   )r3   r   r6   r   rI   )r   r   r   r9   r   r:   r!   r<   s   &&&     r   normalize_youtube_itemsrS      s     J((<(..B&&++g&++g& Z0

 88F#&,,xx
B'((7B'#."5"!#xx(<bAhh{C0."5
 	 2 r   c          	          V ^8  d   QhR\         \        \        \        3,          ,          R\        R\        R\         \        P
                  ,          /# r   )r   r   r   r   r   
TikTokItem)r   s   "r   r   r      sH     , ,S#X,, , 
&

	,r   c                   . p\        V 4       EF2  w  rEVP                  R4      ;'       g    / p\        P                  ! VP                  R4      VP                  R4      VP                  R4      VP                  R4      R7      pVP                  R4      pVP	                  \        P
                  ! RV^,            2VP                  R	R
4      VP                  RR
4      VP                  RR
4      VRVVP                  RR
4      VP                  R. 4      VP                  RR4      VP                  RR
4      R7      4       EK5  	  V# )zNormalize raw TikTok items to schema.

Args:
    items: Raw TikTok items from Apify
    from_date: Start of date range
    to_date: End of date range

Returns:
    List of TikTokItem objects
r!   rK   rA   rL   shares)rK   rA   r#   rW   r   TKrE   r'   r)   author_namerP   caption_snippethashtagsr/   rR   r1   r+   rE   r)   rY   r   r2   r!   rZ   r[   r/   r1   )	enumerater3   r   r6   r   rU   	r   r   r   r9   ir   r:   r!   r<   s	   &&&      r   normalize_tiktok_itemsr`      s    JU#((<(..B&&++g&++g& Z0;;x(	

 88F#&++AaC5z&"%#3"! HH%6;XXj"-hh{C0."5
 	 $6 r   c          	          V ^8  d   QhR\         \        \        \        3,          ,          R\        R\        R\         \        P
                  ,          /# r   )r   r   r   r   r   InstagramItem)r   s   "r   r   r      sH     + +S#X++ + 
&

	+r   c                n   . p\        V 4       EF"  w  rEVP                  R4      ;'       g    / p\        P                  ! VP                  R4      VP                  R4      VP                  R4      R7      pVP                  R4      pVP	                  \        P
                  ! RV^,            2VP                  RR	4      VP                  R
R	4      VP                  RR	4      VRVVP                  RR	4      VP                  R. 4      VP                  RR4      VP                  RR	4      R7      4       EK%  	  V# )zNormalize raw Instagram items to schema.

Args:
    items: Raw Instagram items from ScrapeCreators
    from_date: Start of date range
    to_date: End of date range

Returns:
    List of InstagramItem objects
r!   rK   rA   rL   rM   r   IGrE   r'   r)   rY   rP   rZ   r[   r/   rR   r1   r\   )r]   r3   r   r6   r   rb   r^   s	   &&&      r   normalize_instagram_itemsre      s    JU#((<(..B&&++g&++g& Z0

 88F#&..AaC5z&"%#3"! HH%6;XXj"-hh{C0."5
 	 $4 r   c          	          V ^8  d   QhR\         \        \        \        3,          ,          R\        R\        R\         \        P
                  ,          /# r   )r   r   r   r   r   HackerNewsItem)r   s   "r   r   r   (  sH     6 6S#X66 6 
&

 	6r   c                :   . p\        V 4       EF  w  rEVP                  R4      ;'       g    / p\        P                  ! VP                  R4      VP                  R4      R7      p. pVP                  R. 4       F]  p	VP	                  \        P
                  ! V	P                  R^ 4      RV	P                  RR4      V	P                  R	R4      RR
7      4       K_  	  VP                  R4      p
VP	                  \        P                  ! RV^,            2VP                  RR4      VP                  RR4      VP                  RR4      VP                  RR4      V
RVVVP                  R. 4      VP                  RR4      VP                  RR4      R7      4       EK  	  V# )zNormalize raw Hacker News items to schema.

Args:
    items: Raw HN items from Algolia API
    from_date: Start of date range
    to_date: End of date range

Returns:
    List of HackerNewsItem objects
r!   pointsr#   )r"   r#   r%   Nr&   r'   rE   r*   r   HNr,   r)   hn_urlrP   r.   r/   r0   r1   )r+   r,   r)   rk   r&   r   r2   r!   r%   r.   r/   r1   )r]   r3   r   r6   r   r7   rg   )r   r   r   r9   r_   r   r:   r!   r%   r;   r<   s   &&&        r   normalize_hackernews_itemsrl   (  sm    JU#((<(..B&&++h' ^4

 ."-AeeHa(uuXr*fb)!  . 88F#&//AaC5z((7B'#88Hb)88Hb)"!%!XX&8"=hh{C0."5
 	- $J r   c          	          V ^8  d   QhR\         \        \        \        3,          ,          R\        R\        R\         \        P
                  ,          /# r   )r   r   r   r   r   PolymarketItem)r   s   "r   r   r   a  sH     + +S#X++ + 
&

 	+r   c                   . p\        V 4       EF4  w  rEVP                  R4      ;'       g    VP                  RR4      p\        P                  ! VVP                  RR4      R7      pVP                  R4      pVP	                  \        P
                  ! RV^,            2VP                  RR	4      VP                  R
R	4      VP                  RR	4      VP                  R. 4      VP                  R^ 4      VP                  R4      VRVVP                  R4      VP                  RR4      VP                  RR	4      R7      4       EK7  	  V# )zNormalize raw Polymarket items to schema.

Args:
    items: Raw Polymarket items from Gamma API
    from_date: Start of date range
    to_date: End of date range

Returns:
    List of PolymarketItem objects
	volume1mo
volume24hrg        	liquidity)volumerr   r   PMr,   r'   questionr)   outcome_pricesoutcomes_remainingprice_movementrP   end_dater/   r0   r1   )r+   r,   ru   r)   rv   rw   rx   r   r2   r!   ry   r/   r1   )r]   r3   r   r6   r   rn   )	r   r   r   r9   r_   r   rs   r!   r<   s	   &&&      r   normalize_polymarket_itemsrz   a  s"    JU#+&EE$((<*E&&hh{C0


 88F#&//AaC5z((7B'XXj"-#88$4b9#xx(<a@88$45"!XXj)hh{C0."5
 	 $4 r   c                h    V ^8  d   QhR\         R\         \        \        \        3,          ,          /# )r   r   r   )r   r   r   r   )r   s   "r   r   r     s'     . .$ .4S#X#7 .r   c                J    V  Uu. uF  qP                  4       NK  	  up# u upi )z5Convert schema items to dicts for JSON serialization.)to_dict)r   r   s   & r   items_to_dictsr~     s    ',-utLLNu---s    N)F)__doc__typingr   r   r   r   r   r'   r   r   r   r?   WebSearchItemrI   rU   rb   rg   rn   r	   r   r=   rG   rS   r`   re   rl   rz   r~    r   r   <module>r      s    8 2 2 C""FLL&2F2FHZHZ\b\m\mou  pD  pD  FL  F[  F[  ]c  ]r  ]r  s%P9x-`*Z,^+\6r+\.r   