Dcc@sdZddlmZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZmZddlmZddlmZddlmZddlmZdd lmZdd lmZdd lmZdd lmZdd lmZddlmZddl m!Z!ej"Z"ej#Z#e$Z%ej&ryddl%Z%Wne'k rnXndZ(dZ)ej*dkr7yddl+Z+Wne'k rnNXe+j,e+j-\Z.Z/e0e/e1e.ddZ2e+j3e+j-e2e/fnye4e5fWne6k rddZ4dZ5nXe$dZ7dZ8dZ9dZ:dZ;e$dZ<ej=dZ>ej=dej?Z@ej=dZAej=d ej?ZBej=d!ZCej=d"ej?ZDej=d#ej?ZEej=d$ZFiZGiZHe$e$e$d%ZId&e jJfd'YZJd(e jKfd)YZLdS(*slHyperArch: Pipermail archiving for Mailman - The Dragon De Monsyne TODO: - Should be able to force all HTML to be regenerated next time the archive is run, in case a template is changed. - Run a command to generate tarball of html archives for downloading (probably in the 'update_dirty_archives' method). i(t nested_scopesN(t decode_headert make_header(tHeaderParseError(tCharset(tmm_cfg(tUtils(tErrors(tLockFile(tMailList(ti18n(tSafeDict(tsyslog(tArchiverMailboxts tdarwiniiiicCsKd d d d f}x&|D]\}}|j||}qWtj||S( Nt&s&ts>t"s"(Rs&(Rs<(Rs>(Rs"(treplaceRt uncanonstr(tstlangtreplstthingtrepl((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyt html_quotecs cCs tj|S(N(turllibtquote(R((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyt url_quotemscCs|jddS(Nst (R(R((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyt null_to_spaceqscCsytjj|}Wn2tk rJ}|jtjkr@ntdSX|dkrtj}ztj |td}Wdtj |X|S|dkrd|dSd|dS(Nssize not availableis %(size)i bytes i@Bs %d KB s %d MB ( tostpathtgetsizetOSErrorterrnotENOENTt_R tget_translationt set_languagetset_translation(tfilenameRtsizetetotranstout((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pytsizeofus       s@cCsUt|tjr$tj|}ntjt|}tj|jdd|S(NRs"(t isinstancettypest UnicodeTypeRtwebsafetstrRR(targRR((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyt CGIescapess ([(].*[)])s\s*RE\s*(\[\d+\]\s*)?:\s*s([-+,.\w]+@[-+.\w]+)s&([a-z]+://.*?)(?:_\s|_$|$|[]})>\'"\s])s^\s*$s^\s*\s*$s^\s*\s*$s^([>|:]|>)+c Cs|dkrd}n |j}|dkrN|dkrBtj}qN|j}n|||f}tj|}|rtj|}n|dks|dkrtj |d|dt d|\}}|t|<|t|Rtoutput_charsetR1RFtencodetget_content_charsettstript get_payloadRCtbinasciitErrorREt LookupErrort splitlinestbodytdecode_headers(tselftmessagetsequencet keepHeadersRR9titresultR.RgRhtcsettcset_outtcharsetRstl((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyt__init__sf        %     ,cCsr|jj|}|snytj|dd}Wn*tjk r]}tdd||dSX||j|R_RR=R;RhRi(RuRRL((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyt __setstate__ys  cCs(t|dddkr$||_ndS(NR`(tgetattrR;R`(RuR9((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pytsetListIfUnsetscCst||jS(N(RR_(Rutbuf((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyRscCs|j|j}|j|j}|rb||jd<|j|j}|rb||jds\g<2>NR]tstripped(tdecode_charsetRbR]RiRcRRaR R(R)R_RFR'RRGRdReR*t strip_subject(RuRbR]RcR.tatmark((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyRts$    ! cCs|jjj}|rltj|}dj|jd}tjdd|}tj|d|}n|j}tj dtj }|jd|}tjdd|}|S(Nt%s\%s%\d*ds \s*\d+\s*Rs^((RE|AW|SV|VS)(\[\d+\])?:\s*)+s\s( R`tsubject_prefixRmRdtescapetjointsplitRetlstriptcompiletI(RuR]tprefixt prefix_patt strip_patR((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyRs cCsy"t|}t|j}WnYttttfk r}tj|j j }|dkrhd}nt ||d}nXdj |j S(Nsus-asciis iso-8859-1Ru(RRt __unicode__RqRERIRRRGR`R>RFRRr(RutfieldtpairstustrR{((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyRs   cCs|jj}tj}tj|jz|j\|d<|d<|j\|d<|d<|j|j |d<|j|j |d<|j|j |d<|j|j |d    %  cCs{|jrg|j|j}dt|jj}dtddt|jj|j|f}n d}}||fS(s4Return the href and subject for the previous messages s
  • sPrevious message (by thread):s %s
    • R(RYt_get_subject_encRR+R'R(RuR]RYR((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyRs   cCs|jjd|jS(sReturn the subject of art, decoded if possible. If the charset of the current message and art match and the article's subject is encoded, decode it. R](RiR@R](Rutart((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyR scCs{|jrg|j|j}dt|jj}dtddt|jj|j|f}n d}}||fS(s4Return the href and subject for the previous messagess
  • sNext message (by thread):s %s
    • R(RZRRR+R'R(RuR]RZR((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyRs   s=([A-F0-9][A-F0-9])s=[ ]*$cCs=y |j}Wntk r)|j}nXttj|S(s<Return the message body ready for HTML, decoded if necessary(t html_bodytAttributeErrorRsR t EMPTYSTRINGR(RuRs((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyR"s    cCsVxOd d d d fD];\}}|jj|r|j|j|||s\g<2>s s (RRR@RmttimeRtappendRRRsRRGR_R1R2R3RFRRaR R(R)R'RdReR*RktNL(RuRtheadersRsR{R.R((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pytas_text3s>        cCs,|j|tjt|j|_dS(N(t_Article__super_set_dateRRRRR(RuRv((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyt _set_dateZs cCs}g|_d}xg|j}|s(Pn|sO|jdkrd}qqn|jdkrePn|jj|qWdS(Nisis(RstreadlineRmR(Rutfileobjtbegintline((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pytloadbody_fromHTML^s   cCs+g|_y |`Wntk r&nXdS(N(RsRR(Ru((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pytfinished_update_articlems    N(%t__name__t __module__t pipermailRTRRXRRRt_last_article_timeR;RR=tweakreftWeakValueDictionaryRRRRRRRtRRRRRRRdRt _rx_quotet _rx_softlineRRRRR(((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyRTs4    L          *     '  t HyperArchivecBseZejjZejjZejjZ ejj Z dZ dZ dZdZdZdZdZdZdZdZdZdZdZd Zd Zd Zd Zd ZdZdZdZdZ dZ!dZ"dZ#dZ$dZ%dZ&dZ'dZ(dZ)dZ*dZ+dZ,dZ-dZd(Z.d#Z/d$Z0d%Z1d&Z2d'Z3RS()iiitthreadtmonthiicCsv|j}tj||}|j|ddd|||_d|_|j|_tj |j|_ t |jdr|jj dkrd|_ q|jj dkrd|_ q|jj d krd |_ q|jj d krd |_ qd |_ nd}d}d}id|dd6d|dd6d|dd 6d|||d 6d|||dd 6|_dS(Ntreloaditdatabasetarchive_volume_frequencyityearitquarteritweekitdayRs(?P[0-9]{4,4})s(?P[01][0-9])s(?P[0123][0-9])t^t$sq(?P[1234])$s-(?P[a-zA-Z]+)$s ^Week-of-Mon-(t archive_dirt HyperDatabaset_HyperArchive__super_inittmaillistR;t _lock_fileR>RRRGR}RRtARCHIVE_PERIODt_volre(RuRtdirtdbtyretmretdre((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyRs4         cCs"t||d|jjd|jS(NRR9(RTRR>(RutmsgRw((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyt _makeArticles  cs]j}tj}tj|jfd}zi|jd6|jd6|jdddd6jd6t |j j d6}it d d 6t d d 6t d d 6t d d 6}Wdtj |Xxh|jD]Z}|d j|d}j|kr*d|d|[ %s ]sarchidxfoot.htmlR9(RR R(R)R>RRRRRRRR'R*tkeystupperttypeRS(RuR9R.RRRytttcap((Rus9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyt html_foots0      csj}tj}tj|jfd}zit|jjd6jd6j j d6|j dddd6|j d6|j d 6jd 6}itd d 6td d 6td d 6tdd6}Wdtj|Xxv|jD]h}|dj|d}j|krUd|d|<|||d[ %s ]Rsarchidxhead.htmlR9(RR R(R)R>RRRRt volNameToDescRRRRR,R'R*RRR}RRS(RuR9R.RRRyRR((Rus9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyt html_heads<         c Cs|j}|j}tjj|jd|d}i|jd6|jdddd6d||fd6t||j d6d d 6}t j }t j |j z|j std |d Currently, there are no archives.

    t noarchive_msgtarchive_listing_starttarchive_listing_endtarchive_listingsarchliststart.htmlRR9sarchlistend.htmls archtoc.htmlsarchtocnombox.html(RRR!R"RRRRR0R>R R(R)tarchivesR'RSRthtml_TOC_entryRR*RRRGRt PUBLIC_MBOX( RuR9RLtmboxRR.taccumtaRO((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pythtml_TOCsF  #               cCstjj|jj|d}|d}tjj|re|}|d}dtdd}n?tjj|r|}|d}dtdd}nd}|r|i|d6t||jj d 6}nd }t d i|d 6|j |d 6|d6d|jS(Ns.txts.gzs.txt.gzs[ sGzip'd Text%(sz)ss ]s Text%(sz)sturltszRsarchtocentry.htmlRt archivelabelttextlinkR9( R!R"RRRtexistsR'R;R0R>RSR(RutarchttxtfiletgzfiletfileRttemplR((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyR-s,"     cCsq|jr dStjtjjtj|jjd|_y|jj ddWntj k rldSXdS(Nis -arch.lockttimeoutg?i( RRR!R"RRtLOCK_DIRRRRt TimeOutError(Ru((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyt GetArchLockOs cCs,|jr(|jjddd|_ndS(Ntunconditionallyi(RtunlockR;(Ru((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyt DropArchLock[s cCs|jj}|d}|d}ytj|Wnttjfk rQdSX|jsbdSyt|}tdd||tj d}zt|d}Wdtj |X|j dd|j dd kr|j d n|j |j |j |j tj|Wntk r4nXtj||t|}|j||j tj||jdS( Ns.workings.err_unarchivedR:sGArchive working file %s present. Check %s for possibly unarchived msgsisa+iis (RtArchiveFileNameR!tstattIOErrorR:R$topenR tumasktseektreadtwritetclosetunlinktrenametprocessUnixMailboxR'(Rutnametwnametenametwftomaskteftarchfile((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pytprocessListArch`sB            cCsd|jfS(Ns %06i.html(Rw(Rutarticle((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyt get_filenamescCs@|jt|j}|jtd|j|d|S(sReturn a list of indexes where the article should be filed. A string can be returned if the list only contains one entry, and the empty list is legal.sfiguring article archives s (t dateToVolNametfloatRRvtC_(RuR<tres((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyt get_archivessc Csx|j}dtdtdtdtdtdtdtdtd td td td td g }x|jjD]}tj|j||}|rt|jd}|dkr'dtdtdtdtdg}|t|jd}tdS|dkr|jdj}xpt ddD]_} t j dd| dddddddf } |j| jkrX|| } tdSqXWt d|qc|dkr|t|jd} t|jd} tdS|dkrS|t|jd} t|jd} td S|jdSqqWt d!|dS("NRtJanuarytFebruarytMarchtApriltMaytJunetJulytAugustt SeptembertOctobertNovembertDecemberRRtFirsttSecondtThirdtFourths%(ord)s quarter %(year)iRii s%Biis%(month)s %(year)is%s is not a month!RRs-The Week Of Monday %(day)i %(month)s %(year)is%(day)i %(month)s %(year)is%s is not a valid volname( RmR'RRRdR\RtgroupRftrangeRtstrftimeRI( Rutvolnamet monthdictteachR\RRtordtmonthstrRyt monthnameRR((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyRs> $$* -  -     cCs4tj|}|jdkr.tjd|S|jdkr|ddkr]tjd|S|ddkr}tjd |S|ddkrtjd|Stjd|Sn|jdkrtjd|S|jdkr tj||d ddd}tj|}tjd|Stjd|SdS(NRs%YRiiis%Yq1iiis%Yq2iii s%Yq3s%Yq4Rs%Y%m%dRii<sWeek-of-Mon-%Y%m%ds%Y-%B(iii(iii(iii (Rt localtimeRRUtmktime(RuRt datetupletmonday((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyR>s$#c Cs|j}x|jjD]}tj|j||}|rt|jd}d}d}|dkrt|jd}|dd}n|dkr=|jdj}g} xQtddD]@} | j t j dd | dd d d d dd f jqWy| j |d}Wqt k r9qXnE|d ksU|d krt|jd}t|jd }ny,t j||dd d d d dd f SWqtk rdSXqqWdS(NRiRiiRi s%BiiRRig(RmRRRdR\RRSRfRTRRRUtindexRIR]t OverflowError( RuRVRXR\RRRtqRZtmRy((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyt volNameToDates8   8 ,  csOfd}jdkr1jj|n jjjjdS(NcsFj|}j|}||kr.dS||kr>dSdSdS(Niii(Rd(Rtbtaltbl(Ru(s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pytsfs  RRR(RRR(RRtsorttreverse(RuRh((Rus9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyt sortarchivess   cCsP|jrLtj}|j||ddkr?|jdn|jndS(Nis (tVERBOSEtsyststderrR/tflush(RuRtf((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyRv s    cCsKtjj|d}ytj|WnnXtj|jd|dS(Ns index.htmls.html(R!R"RR1tsymlinkt DEFAULTINDEX(RuRt archivedirt index_html((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pytopen_new_archives cCsQd|_|jGH|j rM|jdkrM|jtd|jndS(NitThreadsComputing threaded index (tdepthR t THREADLAZYRRvR@tupdateThreadedIndex(Ru((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pytwrite_index_headers   cCs.xt|jD] }dGHqW|jGHdS(Ns(RTRwR(RuRy((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pytwrite_index_footer!s cCs|jd|}|jd|}tjrcytjdtd|}Wqctk r_qcXnt||j}t||j}it j |j d6|d6|j d6|d6}t d|d|jGHdS( NR]RbRUs at R+Rwsarchidxentry.htmlR9(t get_headerRRaRdReR'RER7RRRR+RwRSR(RuR<R]RbR((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pytwrite_index_entry&s"     cCs/|jj|}|dkr+t||S|S(N(RiR@R;R(RuRR<Rz((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyR|<s  cCs|dkr"|jdd}n||jkr=|j}n||jkrrxXt|j|D] }dGHq`Wn5||jkrx#t||jD] }dGHqWnd||jfGH||_|j|dS(Nisdepth<0ss
      s (Rvt THREADLEVELSRwRTt threadKeyR}(RuR<RwRy((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pytwrite_threadindex_entryDs       cCsp|jtjd}z%ttjj|jdd}Wdtj|X|j|j|j dS(Nis index.htmltw( RkR!R,R+R"RtbasedirR/RR0(RuR8ttoc((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyt write_TOCTs %cCstjd}zt|d}Wdtj|X|j|j|jtjj|jd|}tjd}zt|d}Wdtj|X|j|j |jdS(NiRs%s.txtsa+( R!R,R+R/RR0R"RRR(RuR`R<R"R8Rp((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyt write_article^s c Cs|j|trd}d}tjj|jd|}tjj|jd|}tjj|jd|}yt|}Wntk rdSXy#tj ||tj|}Wntt tj fk rnXz%tj d}tj|d}Wdtj |X|rD|j |j|jtj|ny+|j |j|j|jWntk rnXtj|ndS(Ns%s.txts %s.txt.gzs %s.old.txt.gziR(t#_HyperArchive__super_update_archivetgzipR;R!R"RRR+R*R2t RuntimeErrorR:R,R/R.R0R1( RuRtarchztarchtRtgzipfiletoldgziptoutnewz((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pytupdate_archivers>     RRR}cCsYi}xL|jjD];}||jkp:|j|ks|j|||ss
      iiRURRsmailto:sj==k: This can't happen!s%s%s(R'R1R2R3RFRRGRtxrangetlenR;tIQUOTESt quotedpatR\R^R7tSHOWHTMLtemailpattsearchturlpattstartRSRRaRdReRRRIR(RutlinesRtsourcetdesttlast_line_was_quotedRytLorigtLRtsuffixtquotedtL2tjrtkrtjtkRPtlengthtURLtpos((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyt__processbody_URLquotesv !             $     $     cCs|}|}t|}d}x||krx3||krctj||dkrc|d}q1W||krd||<|d}nxL||krtj||dkr||d||<||<|d}qW||kr"d||<|d}q"q"WdS(Nii(RthtmlpatR\R;t nohtmlpat(RuRRRR~Ry((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyt__processbody_HTMLs  (   (  cCstd|j}|jr+|j|n|j||j rh|rh|jdd|jdnl|j rt d|}nNxKt dt |D]4}||}|dd!dkrd|||scSs|dS(Ns
      ((tx((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pytRis s

      (tfilterR;Rst ALLOWHTMLt_HyperArchive__processbody_HTMLt#_HyperArchive__processbody_URLquoteRtinsertRtSHOWBRtmapRTRR(RuR<RRyR((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pytformat_article s     c Cs|j}tjj||j}|jtdy't|}|j||j Wn>t k r}|j t j krn|jtdnX||_ ||_tjd} zt|d}Wdtj| X|j|j|j dS(Ns!Updating HTML for article %(seq)ss%article file %(filename)s is missing!iR(RwR!R"RR+RvR@R+RR0R*R%R&RYRZR,R/R( RutarcdirR<RYRZtseqR+RpR-R8((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pytupdate_article$s&     (RRR}(4RRRtTRRRRtupdate_dirty_archivest*_HyperArchive__super_update_dirty_archivest add_articlet _HyperArchive__super_add_articletDIRMODEtFILEMODERlRrRRxR~RRRRRRR RRR$R'R;R=RBRR>RdRkRvRuRzR{R}R|RRRRRRRRR(((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyRus\     %  " ) - "  ,  )            ( Q  (Mt__doc__t __future__RRmRdR%RRR!R2RRRRot email.HeaderRRt email.ErrorsRt email.CharsetRtMailmanRRRRR R tMailman.SafeDictR tMailman.Logging.SyslogR tMailman.MailboxR R'R@R;RtGZIP_ARCHIVE_TXT_FILESt ImportErrorRRtplatformtresourcet getrlimitt RLIMIT_STACKtsoftthardtmintmaxtnewsoftt setrlimitRCtFalset NameErrorRRR R0RR7Rtparen_name_patt IGNORECASER[RRtblankpatRRRR?RARSRTRR(((s9/export/web/mailman/mailman/Mailman/Archiver/HyperArch.pyts                       <u