/[gentoo-src]/portage/pym/portage_util.py
Gentoo

Contents of /portage/pym/portage_util.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.22 - (show annotations) (download) (as text)
Sat Jun 11 09:20:36 2005 UTC (9 years, 6 months ago) by ferringb
Branch: MAIN
CVS Tags: HEAD
Branch point for: portage_2_1
Changes since 1.21: +8 -8 lines
File MIME type: text/x-python
someone needs a wedgie.  stick to tabs, not spaces for indentation (or convert entirely to spaces, either way, one or the other, not
both) :)

1 # Copyright 2004 Gentoo Foundation
2 # Distributed under the terms of the GNU General Public License v2
3 # $Header: /var/cvsroot/gentoo-src/portage/pym/portage_util.py,v 1.21 2005/03/08 12:04:31 ferringb Exp $
4 cvs_id_string="$Id: portage_util.py,v 1.21 2005/03/08 12:04:31 ferringb Exp $"[5:-2]
5
6 import sys,string,shlex,os.path,stat,types
7 import shutil
8
9 try:
10 #XXX: This should get renamed to bsd_chflags, I think.
11 import chflags
12 bsd_chflags = chflags
13 except SystemExit, e:
14 raise
15 except:
16 # XXX: This should get renamed to bsd_chflags, I think.
17 bsd_chflags = None
18
19 noiselimit = 0
20 def writemsg(mystr,noiselevel=0):
21 """Prints out warning and debug messages based on the noiselimit setting"""
22 global noiselimit
23 if noiselevel <= noiselimit:
24 sys.stderr.write(mystr)
25 sys.stderr.flush()
26
27 def grabfile(myfilename, compat_level=0):
28 """This function grabs the lines in a file, normalizes whitespace and returns lines in a list; if a line
29 begins with a #, it is ignored, as are empty lines"""
30
31 try:
32 myfile=open(myfilename,"r")
33 except IOError:
34 return []
35 mylines=myfile.readlines()
36 myfile.close()
37 newlines=[]
38 for x in mylines:
39 #the split/join thing removes leading and trailing whitespace, and converts any whitespace in the line
40 #into single spaces.
41 myline=string.join(string.split(x))
42 if not len(myline):
43 continue
44 if myline[0]=="#":
45 # Check if we have a compat-level string. BC-integration data.
46 # '##COMPAT==>N<==' 'some string attached to it'
47 mylinetest = string.split(myline, "<==", 1)
48 if len(mylinetest) == 2:
49 myline_potential = mylinetest[1]
50 mylinetest = string.split(mylinetest[0],"##COMPAT==>")
51 if len(mylinetest) == 2:
52 if compat_level >= int(mylinetest[1]):
53 # It's a compat line, and the key matches.
54 newlines.append(myline_potential)
55 continue
56 else:
57 continue
58 newlines.append(myline)
59 return newlines
60
61 def stack_dictlist(original_dicts, incremental=0, incrementals=[], ignore_none=0):
62 """Stacks an array of dict-types into one array. Optionally merging or
63 overwriting matching key/value pairs for the dict[key]->list.
64 Returns a single dict. Higher index in lists is preferenced."""
65 final_dict = None
66 kill_list = {}
67 for mydict in original_dicts:
68 if mydict == None:
69 continue
70 if final_dict == None:
71 final_dict = {}
72 for y in mydict.keys():
73 if not final_dict.has_key(y):
74 final_dict[y] = []
75 if not kill_list.has_key(y):
76 kill_list[y] = []
77
78 mydict[y].reverse()
79 for thing in mydict[y]:
80 if thing and (thing not in kill_list[y]) and ("*" not in kill_list[y]):
81 if (incremental or (y in incrementals)) and thing[0] == '-':
82 if thing[1:] not in kill_list[y]:
83 kill_list[y] += [thing[1:]]
84 else:
85 if thing not in final_dict[y]:
86 final_dict[y].append(thing[:])
87 mydict[y].reverse()
88 if final_dict.has_key(y) and not final_dict[y]:
89 del final_dict[y]
90 return final_dict
91
92 def stack_dicts(dicts, incremental=0, incrementals=[], ignore_none=0):
93 """Stacks an array of dict-types into one array. Optionally merging or
94 overwriting matching key/value pairs for the dict[key]->string.
95 Returns a single dict."""
96 final_dict = None
97 for mydict in dicts:
98 if mydict == None:
99 if ignore_none:
100 continue
101 else:
102 return None
103 if final_dict == None:
104 final_dict = {}
105 for y in mydict.keys():
106 if mydict[y]:
107 if final_dict.has_key(y) and (incremental or (y in incrementals)):
108 final_dict[y] += " "+mydict[y][:]
109 else:
110 final_dict[y] = mydict[y][:]
111 mydict[y] = string.join(mydict[y].split()) # Remove extra spaces.
112 return final_dict
113
114 def stack_lists(lists, incremental=1):
115 """Stacks an array of list-types into one array. Optionally removing
116 distinct values using '-value' notation. Higher index is preferenced."""
117 new_list = []
118 for x in lists:
119 for y in x:
120 if y:
121 if incremental and y[0]=='-':
122 while y[1:] in new_list:
123 del new_list[new_list.index(y[1:])]
124 else:
125 if y not in new_list:
126 new_list.append(y[:])
127 return new_list
128
129 def grab_multiple(basename, locations, handler, all_must_exist=0):
130 mylist = []
131 for x in locations:
132 mylist.append(handler(x+"/"+basename))
133 return mylist
134
135 def grabdict(myfilename,juststrings=0,empty=0):
136 """This function grabs the lines in a file, normalizes whitespace and returns lines in a dictionary"""
137 newdict={}
138 try:
139 myfile=open(myfilename,"r")
140 except IOError,e:
141 return newdict
142 mylines=myfile.readlines()
143 myfile.close()
144 for x in mylines:
145 #the split/join thing removes leading and trailing whitespace, and converts any whitespace in the line
146 #into single spaces.
147 if x[0] == "#":
148 continue
149 myline = x.split()
150 if len(myline)<2 and empty==0:
151 continue
152 if len(myline)<1 and empty==1:
153 continue
154 if juststrings:
155 newdict[myline[0]]=" ".join( myline[1:] )
156 else:
157 newdict[myline[0]]=myline[1:]
158 return newdict
159
160 def grabdict_package(myfilename,juststrings=0):
161 from portage_dep import isvalidatom
162 pkgs=grabdict(myfilename, juststrings, empty=1)
163 for x in pkgs.keys():
164 if not isvalidatom(x):
165 del(pkgs[x])
166 writemsg("--- Invalid atom in %s: %s\n" % (myfilename, x))
167 return pkgs
168
169 def grabfile_package(myfilename,compatlevel=0):
170 from portage_dep import isvalidatom
171 pkgs=grabfile(myfilename,compatlevel)
172 for x in range(len(pkgs)-1,-1,-1):
173 pkg = pkgs[x]
174 if pkg[0] == "-":
175 pkg = pkg[1:]
176 if pkg[0] == "*": # Kill this so we can deal the "packages" file too
177 pkg = pkg[1:]
178 if not isvalidatom(pkg): #XXX: isvalidatom is not available from this context
179 writemsg("--- Invalid atom in %s: %s\n" % (myfilename, pkgs[x]))
180 del(pkgs[x])
181 return pkgs
182
183 def grabints(myfilename):
184 newdict={}
185 try:
186 myfile=open(myfilename,"r")
187 except IOError:
188 return newdict
189 mylines=myfile.readlines()
190 myfile.close()
191 for x in mylines:
192 #the split/join thing removes leading and trailing whitespace, and converts any whitespace in the line
193 #into single spaces.
194 myline=x.split()
195 if len(myline)!=2:
196 continue
197 newdict[myline[0]]=int(myline[1])
198 return newdict
199
200 def writeints(mydict,myfilename):
201 try:
202 myfile=open(myfilename,"w")
203 except IOError:
204 return 0
205 for x in mydict.keys():
206 myfile.write(x+" "+`mydict[x]`+"\n")
207 myfile.close()
208 return 1
209
210 def writedict(mydict,myfilename,writekey=1):
211 """Writes out a dict to a file; writekey=0 mode doesn't write out
212 the key and assumes all values are strings, not lists."""
213 try:
214 myfile=open(myfilename,"w")
215 except IOError:
216 writemsg("Failed to open file for writedict(): "+str(myfilename)+"\n")
217 return 0
218 if not writekey:
219 for x in mydict.values():
220 myfile.write(x+"\n")
221 else:
222 for x in mydict.keys():
223 myfile.write(x+" ")
224 for y in mydict[x]:
225 myfile.write(y+" ")
226 myfile.write("\n")
227 myfile.close()
228 return 1
229
230 def getconfig(mycfg,tolerant=0,allow_sourcing=False):
231 mykeys={}
232 try:
233 f=open(mycfg,'r')
234 except IOError:
235 return None
236 try:
237 lex=shlex.shlex(f)
238 lex.wordchars=string.digits+string.letters+"~!@#$%*_\:;?,./-+{}"
239 lex.quotes="\"'"
240 if allow_sourcing:
241 lex.source="source"
242 while 1:
243 key=lex.get_token()
244 if (key==''):
245 #normal end of file
246 break;
247 equ=lex.get_token()
248 if (equ==''):
249 #unexpected end of file
250 #lex.error_leader(self.filename,lex.lineno)
251 if not tolerant:
252 writemsg("!!! Unexpected end of config file: variable "+str(key)+"\n")
253 raise Exception("ParseError: Unexpected EOF: "+str(mycfg)+": on/before line "+str(lex.lineno))
254 else:
255 return mykeys
256 elif (equ!='='):
257 #invalid token
258 #lex.error_leader(self.filename,lex.lineno)
259 if not tolerant:
260 writemsg("!!! Invalid token (not \"=\") "+str(equ)+"\n")
261 raise Exception("ParseError: Invalid token (not '='): "+str(mycfg)+": line "+str(lex.lineno))
262 else:
263 return mykeys
264 val=lex.get_token()
265 if (val==''):
266 #unexpected end of file
267 #lex.error_leader(self.filename,lex.lineno)
268 if not tolerant:
269 writemsg("!!! Unexpected end of config file: variable "+str(key)+"\n")
270 raise portage_exception.CorruptionError("ParseError: Unexpected EOF: "+str(mycfg)+": line "+str(lex.lineno))
271 else:
272 return mykeys
273 mykeys[key]=varexpand(val,mykeys)
274 except SystemExit, e:
275 raise
276 except Exception, e:
277 raise e.__class__, str(e)+" in "+mycfg
278 return mykeys
279
280 #cache expansions of constant strings
281 cexpand={}
282 def varexpand(mystring,mydict={}):
283 """
284 new variable expansion code. Removes quotes, handles \n, etc.
285 This code is used by the configfile code, as well as others (parser)
286 This would be a good bunch of code to port to C.
287 """
288 mystring=" "+mystring
289 if mystring in cexpand:
290 return cexpand[mystring]
291 numvars=0
292 #in single, double quotes
293 insing=0
294 indoub=0
295 pos=1
296 newstring=" "
297 while (pos<len(mystring)):
298 if (mystring[pos]=="'") and (mystring[pos-1]!="\\"):
299 if (indoub):
300 newstring=newstring+"'"
301 else:
302 insing=not insing
303 pos=pos+1
304 continue
305 elif (mystring[pos]=='"') and (mystring[pos-1]!="\\"):
306 if (insing):
307 newstring=newstring+'"'
308 else:
309 indoub=not indoub
310 pos=pos+1
311 continue
312 if (not insing):
313 #expansion time
314 if (mystring[pos]=="\n"):
315 #convert newlines to spaces
316 newstring=newstring+" "
317 pos=pos+1
318 elif (mystring[pos]=="\\"):
319 #backslash expansion time
320 if (pos+1>=len(mystring)):
321 newstring=newstring+mystring[pos]
322 break
323 else:
324 a=mystring[pos+1]
325 pos=pos+2
326 if a=='a':
327 newstring=newstring+chr(007)
328 elif a=='b':
329 newstring=newstring+chr(010)
330 elif a=='e':
331 newstring=newstring+chr(033)
332 elif (a=='f') or (a=='n'):
333 newstring=newstring+chr(012)
334 elif a=='r':
335 newstring=newstring+chr(015)
336 elif a=='t':
337 newstring=newstring+chr(011)
338 elif a=='v':
339 newstring=newstring+chr(013)
340 elif a!='\n':
341 #remove backslash only, as bash does: this takes care of \\ and \' and \" as well
342 newstring=newstring+mystring[pos-1:pos]
343 continue
344 elif (mystring[pos]=="$") and (mystring[pos-1]!="\\"):
345 pos=pos+1
346 if mystring[pos]=="{":
347 pos=pos+1
348 braced=True
349 else:
350 braced=False
351 myvstart=pos
352 validchars=string.ascii_letters+string.digits+"_"
353 while mystring[pos] in validchars:
354 if (pos+1)>=len(mystring):
355 if braced:
356 cexpand[mystring]=""
357 return ""
358 else:
359 pos=pos+1
360 break
361 pos=pos+1
362 myvarname=mystring[myvstart:pos]
363 if braced:
364 if mystring[pos]!="}":
365 cexpand[mystring]=""
366 return ""
367 else:
368 pos=pos+1
369 if len(myvarname)==0:
370 cexpand[mystring]=""
371 return ""
372 numvars=numvars+1
373 if mydict.has_key(myvarname):
374 newstring=newstring+mydict[myvarname]
375 else:
376 newstring=newstring+mystring[pos]
377 pos=pos+1
378 else:
379 newstring=newstring+mystring[pos]
380 pos=pos+1
381 if numvars==0:
382 cexpand[mystring]=newstring[1:]
383 return newstring[1:]
384
385 def pickle_write(data,filename,debug=0):
386 import cPickle,os
387 try:
388 myf=open(filename,"w")
389 cPickle.dump(data,myf,cPickle.HIGHEST_PROTOCOL)
390 myf.close()
391 writemsg("Wrote pickle: "+str(filename)+"\n",1)
392 os.chown(myefn,uid,portage_gid)
393 os.chmod(myefn,0664)
394 except SystemExit, e:
395 raise
396 except Exception, e:
397 return 0
398 return 1
399
400 def pickle_read(filename,default=None,debug=0):
401 import cPickle,os
402 if not os.access(filename, os.R_OK):
403 writemsg("pickle_read(): File not readable. '"+filename+"'\n",1)
404 return default
405 data = None
406 try:
407 myf = open(filename)
408 mypickle = cPickle.Unpickler(myf)
409 mypickle.find_global = None
410 data = mypickle.load()
411 myf.close()
412 del mypickle,myf
413 writemsg("pickle_read(): Loaded pickle. '"+filename+"'\n",1)
414 except SystemExit, e:
415 raise
416 except Exception, e:
417 writemsg("!!! Failed to load pickle: "+str(e)+"\n",1)
418 data = default
419 return data
420
421 class ReadOnlyConfig:
422 def __init__(self,filename,strict_keys=0):
423 self.__filename = filename[:]
424 self.__strict_keys = strict_keys
425 self.__mydict = {}
426 self.__dict_was_loaded = False
427 if os.path.isfile(self.__filename):
428 self.__mydict = getconfig(self.__filename)
429 self.__dict_was_loaded = True
430
431 def isLoaded():
432 return self.__dict_was_loaded
433
434 def __getitem__(self,key):
435 if self.__mydict.has_key(key):
436 return self.__mydict[key][:]
437 if self.__strict_keys:
438 raise KeyError("%s not found in config: '%s'" % (key,self.__filename))
439 return ""
440
441 def __setitem__(self,key,value):
442 raise KeyError("This class is not modifiable.")
443
444 def keys(self):
445 return self.__mydict.keys()
446
447 def has_key(self,key):
448 return self.__mydict.has_key(key)
449
450 def unique_array(array):
451 """Takes an array and makes sure each element is unique."""
452 newarray = []
453 for x in array:
454 if x not in newarray:
455 newarray.append(x)
456 return newarray
457
458 def movefile(src,dest,newmtime=None,sstat=None,mysettings=None):
459 """moves a file from src to dest, preserving all permissions and attributes; mtime will
460 be preserved even when moving across filesystems. Returns true on success and false on
461 failure. Move is atomic."""
462 #print "movefile("+str(src)+","+str(dest)+","+str(newmtime)+","+str(sstat)+")"
463 global lchown
464 from portage_exec import selinux_capable
465 if selinux_capable:
466 import selinux
467 from portage_data import lchown
468 try:
469 if not sstat:
470 sstat=os.lstat(src)
471 if bsd_chflags:
472 sflags=bsd_chflags.lgetflags(src)
473 if sflags < 0:
474 # Problem getting flags...
475 print "!!! Couldn't get flags for "+dest+"\n"
476 return None
477
478 except SystemExit, e:
479 raise
480 except Exception, e:
481 print "!!! Stating source file failed... movefile()"
482 print "!!!",e
483 return None
484
485 destexists=1
486 try:
487 dstat=os.lstat(dest)
488 except SystemExit, e:
489 raise
490 except:
491 dstat=os.lstat(os.path.dirname(dest))
492 destexists=0
493
494 if bsd_chflags:
495 # Check that we can actually unset schg etc flags...
496 # Clear the flags on source and destination; we'll reinstate them after merging
497 if(destexists):
498 if bsd_chflags.lchflags(dest, 0) < 0:
499 print "!!! Couldn't clear flags on file being merged: \n"
500 # We might have an immutable flag on the parent dir; save and clear.
501 pflags=bsd_chflags.lgetflags(os.path.dirname(dest))
502 bsd_chflags.lchflags(os.path.dirname(dest), 0)
503
504 # Don't bother checking the return value here; if it fails then the next line will catch it.
505 bsd_chflags.lchflags(src, 0)
506
507 if bsd_chflags.lhasproblems(src)>0 or (destexists and bsd_chflags.lhasproblems(dest)>0) or bsd_chflags.lhasproblems(os.path.dirname(dest))>0:
508 # This is bad: we can't merge the file with these flags set.
509 print "!!! Can't merge file "+dest+" because of flags set\n"
510 return None
511
512 if destexists:
513 if stat.S_ISLNK(dstat[stat.ST_MODE]):
514 try:
515 os.unlink(dest)
516 destexists=0
517 except SystemExit, e:
518 raise
519 except Exception, e:
520 pass
521
522 if stat.S_ISLNK(sstat[stat.ST_MODE]):
523 try:
524 target=os.readlink(src)
525 if mysettings and mysettings["D"]:
526 if target.find(mysettings["D"])==0:
527 target=target[len(mysettings["D"]):]
528 if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]):
529 os.unlink(dest)
530 if selinux_capable:
531 sid = selinux.get_lsid(src)
532 selinux.secure_symlink(target,dest,sid)
533 else:
534 os.symlink(target,dest)
535 lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
536 if bsd_chflags:
537 # Restore the flags we saved before moving
538 if bsd_chflags.lchflags(dest, sflags) < 0 or bsd_chflags.lchflags(os.path.dirname(dest), pflags) < 0:
539 writemsg("!!! Couldn't restore flags ("+str(flags)+") on " + dest+":\n")
540 writemsg("!!! %s\n" % str(e))
541 return None
542 return os.lstat(dest)[stat.ST_MTIME]
543 except SystemExit, e:
544 raise
545 except Exception, e:
546 print "!!! failed to properly create symlink:"
547 print "!!!",dest,"->",target
548 print "!!!",e
549 return None
550
551 renamefailed=1
552 if sstat[stat.ST_DEV]==dstat[stat.ST_DEV] or selinux_capable:
553 try:
554 if selinux_capable:
555 ret=selinux.secure_rename(src,dest)
556 else:
557 ret=os.rename(src,dest)
558 renamefailed=0
559 except SystemExit, e:
560 raise
561 except Exception, e:
562 import errno
563 if e[0]!=errno.EXDEV:
564 # Some random error.
565 print "!!! Failed to move",src,"to",dest
566 print "!!!",e
567 return None
568 # Invalid cross-device-link 'bind' mounted or actually Cross-Device
569 if renamefailed:
570 didcopy=0
571 if stat.S_ISREG(sstat[stat.ST_MODE]):
572 try: # For safety copy then move it over.
573 if selinux_capable:
574 selinux.secure_copy(src,dest+"#new")
575 selinux.secure_rename(dest+"#new",dest)
576 else:
577 shutil.copyfile(src,dest+"#new")
578 os.rename(dest+"#new",dest)
579 didcopy=1
580 except SystemExit, e:
581 raise
582 except Exception, e:
583 print '!!! copy',src,'->',dest,'failed.'
584 print "!!!",e
585 return None
586 else:
587 #we don't yet handle special, so we need to fall back to /bin/mv
588 if selinux_capable:
589 a=portage_exec.spawn_get_output(MOVE_BINARY+" -c -f '%s' '%s'" % (src,dest))
590 else:
591 a=portage_exec.spawn_get_output(MOVE_BINARY+" -f '%s' '%s'" % (src,dest))
592 if a[0]!=0:
593 print "!!! Failed to move special file:"
594 print "!!! '"+src+"' to '"+dest+"'"
595 print "!!!",a
596 return None # failure
597 try:
598 if didcopy:
599 lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
600 os.chmod(dest, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown
601 os.unlink(src)
602 except SystemExit, e:
603 os.unlink(src)
604 except SystemExit, e:
605 raise
606 except Exception, e:
607 print "!!! Failed to chown/chmod/unlink in movefile()"
608 print "!!!",dest
609 print "!!!",e
610 return None
611
612 if newmtime:
613 os.utime(dest,(newmtime,newmtime))
614 else:
615 os.utime(dest, (sstat[stat.ST_ATIME], sstat[stat.ST_MTIME]))
616 newmtime=sstat[stat.ST_MTIME]
617
618 if bsd_chflags:
619 # Restore the flags we saved before moving
620 if bsd_chflags.lchflags(dest, sflags) < 0 or bsd_chflags.lchflags(os.path.dirname(dest), pflags) < 0:
621 writemsg("!!! Couldn't restore flags ("+str(sflags)+") on " + dest+":\n")
622 return None
623
624 return newmtime
625
626 def flatten(mytokens):
627 """this function now turns a [1,[2,3]] list into
628 a [1,2,3] list and returns it."""
629 newlist=[]
630 for x in mytokens:
631 if type(x)==list:
632 newlist.extend(flatten(x))
633 else:
634 newlist.append(x)
635 return newlist
636
637
638 def abssymlink(symlink):
639 """
640 This reads symlinks, resolving the relative symlinks, and returning the absolute.
641 """
642 mylink=os.readlink(symlink)
643 if mylink[0] != '/':
644 mydir=os.path.dirname(symlink)
645 mylink=mydir+"/"+mylink
646 return os.path.normpath(mylink)

  ViewVC Help
Powered by ViewVC 1.1.20