
4Jc           @   s  d  d k  l Z d  d k Z d  d k Z d  d k Z d  d k Z d   Z d   Z d   Z d   Z	 d   Z
 d   Z d	   Z d
   Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z e d  Z  e d  Z! e d  Z" e d  Z# e d   Z$ d!   Z% d"   Z& d#   Z' d$   Z( d S(%   i(   t   divisionNc         C   s'   y t  i |   t SWn t Sn Xd  S(   N(   t   ost   statt   Truet   False(   t   path(    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   file_exists   s
    c         C   s   t  } t |  d  } d | } | i   } xC | d j o5 t i | |  o | i   t Sn | i   } q. W| i   t  S(   Nt   rs   .*%s.*t    (   R   t   opent   readlinet   ret   matcht   closeR   (   t   filet   stringt   rvt   handlet   patternt   line(    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   file_contains   s    
 

c         C   s   d |  | f S(   Ns
   %s_nacc_%s(    (   t   prefixt   nacc(    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   get_default_data_dir_name1   s    c         C   s   d |  | f S(   Ns   %s_nacc_%s_a(    (   R   R   (    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   get_default_analysis_dir_name5   s    c         C   s;   t  i d d |  } t  i d d |  } d |  | | f S(   Nt    t   _t   =s   L_%s_T_%s_%s.txt(   R   t   sub(   t   Lt   Tt   aargt   farg(    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   get_RV_file_name9   s    c         C   s   d |  | | f S(   Ns   L_%s_T_%s_rell_alpha0_%s.txt(    (   R   R   t   alpha(    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   get_RV_file_name_alpha?   s    c         C   s;   t  i d d |  } t  i d d |  } d |  | | f S(   NR   R   R   s   L_%s_T_%s_%s.rzn(   R   R   (   R   R   R   R    (    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   get_rzn_file_nameC   s    c         C   s   | } t  |  d j o | d 7} n t i d d |  } h  d d <d d <d	 d
 <d d <d d <d d <d d <d d <d d <} | | } d | | |  S(   Ni   t   0s   \.R   t   00s   0.000t   01s   0.0001t   02s   0.0002t   05s   0.0005t   08s   0.0008t   10s   0.001t   20s   0.002t   30s   0.003t   40s   0.004t   kerl(   t   lenR   R   (   R   R   R"   t   T_codet   alpha_remapst
   alpha_code(    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   encode12N   s     								
c      
   C   s  t  |   d j p |  d d !d j o" t i d |  IJt i d  n |  d d !} |  d d !} |  d d !} | } | d	 d
 j o | d d	 !} n | d d | d } h  d d <d d <d d <d d <d d <d d <d d <d d <d d <} | | } | | | g S(   Ni   i    i   R/   s   Invalid enc12 string %s.i   i   i
   i   R%   t   .s   0.000R&   s   0.0001R'   s   0.0002R(   s   0.0005R)   s   0.0008R*   s   0.001R+   s   0.002R,   s   0.003R-   s   0.004R.   (   R0   t   syst   stderrt   exit(   R   R3   R1   t   L_codeR   R   R2   R"   (    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   decode12a   s,    '								
c         C   s8   t  i d d |   } t  i d d |  } d | | f S(   NR   R   R   s   graph_%s_%s.txt(   R   R   (   R   t   RV_txt_file_nameR    (    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   get_graph_file_name   s    c           C   s   t  i   d S(   Ni   (   R   t   uname(    (    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   get_host_name   s    c         C   sK   t  i |   } | d j p | d j o t i d IJt i d  n | S(   Ni   i   s   Aborted.i   (   R   t   systemR6   R7   R8   (   t   cmdt   rc(    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   system_with_abend   s
    c         C   s5   t  |   p d Sn t |  d  p d Sn d Sd  S(   Ni    t   Elapsedi   (   R   R   (   R   (    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   is_RV_file_complete   s
    c         C   s5   t  |   p d Sn t |  d  p d Sn d Sd  S(   Nt   nRC   t   ft   c(   R   R   (   R   (    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   get_RV_file_status   s
    c   
      C   s   d | t  | | |  f } d } t |  pu | o d | GHn d |  | | | | | f } t |  }	 |	 d j o( t i d |	 | f IJt i d  q n | o d | GHn d  S(   Ns   %s/%si   s   Incomplete s   %s L=%s T=%s %s %s > %si    s    rc %d from command %s.  Exiting.s	   Complete (   R!   RD   RB   R6   R7   R8   (
   t   exet   datadirt   common_argsR   R   R   R   t   chattyR@   RA   (    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   execute_task_unless_complete   s     
 c   	      C   sk   y t  i |  Wn n XxI | D]A } x8 | D]0 } x' | D] } t |  | | | | |  q< Wq/ Wq" Wd  S(   N(   R   t   mkdirRM   (	   RI   RJ   RK   t   Lst   Tst   aargsR   R   R   (    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   execute_single_CPU   s         c         C   s   y t  i |  Wn n Xt |  } g  } x | D]x } t  i   }	 |	 d j oL x8 | D]0 }
 x' | D] } t |  | t |
 | |  qg WqZ Wt i d  q4 | i |	  q4 Wx+ t	 d |  D] } t  i
 | | d  q Wd  S(   Ni    (   R   RN   R0   t   forkRM   RK   R6   R8   t   appendt   ranget   waitpid(   RI   RJ   t   commmon_argsRO   RP   RQ   t   ncput   pidsR   t   pidR   R   t   i(    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   execute_multi_CPU_by_T   s*         	 c         C   sS   g  } xF | D]> } x5 | D]- } x$ |  D] } | i  | | | g  q' Wq Wq W| S(   N(   RT   (   RO   RP   RQ   t   listR   R   R   (    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   flatten_Ls_Ts_aargs   s       "c      
   C   s  y t  i |  Wn n Xd } t | | |  } g  }	 | o) d GHd G| GHd G| GHd G| GHd GHn | o? d GHd | GHd t |  t |  t |  t |  f GHn x5| g  j o'| o$ d GHd	 t |	  t |  f GHn t |	  | j  o | i d  \ }
 } } t  i   } | d j o* t |  | | |
 | |  t i d  q|	 i	 |  | o d
 | |
 | | f GHqq t  i
 d  \ } } } | d j  o d | GHq | o d | | f GHn |	 i |  q Wx |	 g  j o | o$ d GHd	 t |	  t |  f GHn t  i
 d  \ } } } | d j  o d | GHq| o d | | f GHn |	 i |  qW| o d GHd GHn d  S(   Ni    s   ###s   Ls:s   Ts:s   as:R   s$   execute multi CPU enter.  ncpu = %d.s'   #Ls: %d; #Ts: %d; #aargs: %d; total: %ds&   # processes: %d. # unstarted tasks: %ds$   Forked PID %d for L=%s T=%s aarg=%s.s(   wait3 returned %d.  What does that mean?s   PID %d complete with status %d.s   execute multi CPU exit.(   R   RN   R^   R0   t   popRS   RM   R6   R8   RT   t   wait3t   remove(   RI   RJ   RK   RO   RP   RQ   RX   t   verboset   unstarted_taskst	   processesR   R   R   RZ   t   statust   usage(    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   execute_multi_CPU   sl      					.  c         C   sF   d } x3 | D]+ } |  | j o t  | g Sn | d 7} q Wt d g S(   Ni    i   i(   R   R   (   t   elementR]   t   indext   other(    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt
   find_index+  s     c         C   sU  t    } t | |  \ } }	 t |  }
 | pe t i d t i d | f IJt i d Ix | D] } t i d | Iqc Wt i Jt i d  n t | | |  } g  } t |  } x= t d |  D], } | |
 |	 j o | i	 | |  q q Wy t
 i |  Wn n Xx9 | D]1 \ } } } t |  | | | g | g | g  qWd  S(   Ns5   %s: execute_multi_host:  current host %s not in list.i    s   List:s    %si   (   R>   Rk   R0   R6   R7   t   argvR8   R^   RU   RT   R   RN   RR   (   RI   RJ   RK   RO   RP   RQ   t
   host_namest	   host_namet   host_in_listRi   t	   num_hostsRh   t	   all_taskst	   our_taskst	   num_tasksR[   R   R   R   (    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   execute_multi_host4  s6    	     c         C   s   d | t  | | |  f } d | t | | |  f } t |  pd d |  | | | | | | f }	 t |	  }
 |
 d j o( t i d |
 |	 f IJt i d  q n d  S(   Ns   %s/%ss   %s L=%s T=%s %s %s rzn=%s > %si    s    rc %d from command %s.  Exiting.i   (   R!   R$   RD   RB   R6   R7   R8   (   RI   t   data_dirt   rzn_dirRK   R   R   R   t   RV_filet   rzn_fileR@   RA   (    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt    rzn_execute_task_unless_completeU  s    
c   
      C   s   y t  i |  Wn n Xy t  i |  Wn n XxL | D]D } x; | D]3 } x* | D]" }	 t | | |  | | | |	  qW WqJ Wq= Wd  S(   N(   R   RN   Ry   (
   t	   mcrcm_exeRu   Rv   RK   RO   RP   RQ   R   R   R   (    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   rzn_execute_single_CPUh  s            c         C   s   y t  i |  Wn n Xy t  i |  Wn n Xx | D] } x | D] } x | D] } d | t | | |  f }	 d | t | | |  f }
 d |  |	 |
 f } t |  } |	 GH| d j o( t i d | | f IJt i d  qW qW WHqJ WHq= Wd  S(   Ns   %s/%ss   %s rzn=%s > %si    s    rc %d from command %s.  Exiting.i   (   R   RN   R$   R!   RB   R6   R7   R8   (   t	   rvrcm_exeRv   Ru   RO   RP   RQ   R   R   R   Rx   Rw   R@   RA   (    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   rzn_extract_RV_filesu  s2           
c         C   s7  x0| D](} t  i d d |  } t  i d d |  } d |  | f GHx | D] } d } d } d }	 d }
 x | D] } d |  | | | f } t |  p" | d 7} |
 o d | GHq qr t | d	  p" | d 7} |
 o d
 | GHq qr |	 d 7}	 qr W| | |	 } d | | | |	 d |	 | f GHqM Wq Wd  S(   NR   R   R   s   %s %si    s   %s/L_%s_T_%s_%s.txti   s	   Missing  RC   s	   Fragment s:     L=%2s missing %4d; fragment %4d; complete %4d.    %3.f%%g      Y@(   R   R   R   R   (   RJ   RO   RP   RQ   R   R    R   t   nmissingt	   nfragmentt	   ncompletet
   printfilesR   R   t   ntotal(    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   report_progress_by_L  s4       

c         C   s  y t  | d  } Wn8 | o& t i d | d IJt i d  qN d Sn Xd } x | i   } | d j o Pn t i d d |  } d	 |  } t i | |  o7 | i   t i d
 d |  } | i	   } | d SqW qW | o( t i d |  | f IJt i d  n d Sd  S(   NR   s   Couldn't open "s   " for read.i   R   i    R   s   \n+$s   ^#.*%s s   \s+$is   Couldn't find "%s" in "%s".(
   R	   R6   R7   R8   R
   R   R   R   R   t   split(   t   RV_namet	   file_namet   die_on_missingt   file_handlet   got_itR   R   t   fields(    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   get_random_variablef  s0    


c         C   s&   d | t  |   f } t | | |  S(   Ns   %s/%s(   t   task_to_file_nameR   (   t   taskR   Ru   R   R   (    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   get_random_variable  s    c         C   s6   t  d |  | |  } t  d |  | |  } | | g S(   Nt   mean_t   cssm_(   R   (   R   R   R   t   mean_RVt   cssm_RV(    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt#   get_random_variablef_with_error_bar  s
    		c         C   s  y t  i |  Wn n Xt i d d |  } t i d d |  } g  }	 t |  }
 xz t d |
  D]i } | | } g  } xC | D]; } d |  | | | f } t | | |  } | i |  q W|	 i |  qg Wd | | | f } d | Gt |	 d | d	 | |  d
 GHd  S(   NR   R   R   i    s   %s/L_%s_T_%s_%s.txts   %s/graph_%s_%s.txts   Writing %s ...R   R   s   done.(	   R   RN   R   R   R0   RU   R   RT   t   print_RVs_with_labels(   t   ddirt   adirRO   RP   R   t   RV_mcrcm_nameR;   R   R    t   RV_datat   nLt   iLR   t   Li_meansR   t   dfilet   meant
   graph_file(    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   extract_RV_name_pair  s,       
 c         C   s7  y t  i |  Wn n Xt i d d |  } t i d d |  } g  }	 t |  }
 x t d |
  D] } | | } g  } g  } xV | D]N } d |  | | | f } t | | |  \ } } | i |  | i |  q W|	 i |  |	 i |  qg Wd | | | f } d | Gt |	 d | d	 | |  d
 GHd  S(   NR   R   R   i    s   %s/L_%s_T_%s_%s.txts   %s/grapheb_%s_%s.txts   Writing %s ...R   R   s   done.(	   R   RN   R   R   R0   RU   R   RT   t$   print_RVs_with_labels_and_error_bars(   R   R   RO   RP   R   R   R;   R   R    R   R   R   R   R   t   Li_cssmsR   R   R   t   cssmR   (    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt$   extract_RV_name_pair_with_error_bars  s4       
 c   
      C   s   g  } g  } xv | D]n } d |  t  | | |  f } t | | d t }	 |	 d j o* | i t |   | i t |	   q q W| | g S(   Ns   %s/%sR   R   (   R!   R   R   RT   t   float(
   R   R   RP   R   R   t	   output_Tst	   output_XsR   R   t   X(    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   extract_RVs_if_present  s     c   	      C   s}   g  } xp | D]h } d |  t  | | |  f } t | | d t } | d j o | i d   q | i t |   q W| S(   Ns   %s/%sR   R   (   R!   R   R   RT   t   NoneR   (	   R   R   RP   R   R   t   XsR   R   R   (    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyt   extract_RVs#  s     c   	      C   sU   | g |  } | g } x% | D] } | i  d | | f  q Wt i | | |  d  S(   Ns   %s=%s(   RT   t	   tabutil_mt   text_columns_and_labels_to_file(	   R   t   x_namet   x_labelst   y_namet   y_labelsR   t   columnst   labelst   y_label(    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyR   1  s    	 c   	      C   sl   | g |  } | g } x< | D]4 } | i  d | | f  | i  d | | f  q Wt i | | |  d  S(   Ns   %s=%s,ms   %s=%s,e(   RT   R   R   (	   t   RV_and_eb_dataR   R   R   R   R   R   R   R   (    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pyR   =  s    	 ()   t
   __future__R    R6   R   R   R   R   R   R   R   R!   R#   R$   R4   R:   R<   R>   RB   RD   RH   RM   RR   R\   R^   Rg   Rk   Rt   Ry   R{   R}   R   R   R   R   R   R   R   R   R   R   R   (    (    (    s5   /home/kerl/pub_http_internet/rcm/taskutil/taskutil.pys   <module>   sH   $																			
				J			!				"
 			