1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42 """
43 Provides general-purpose utilities.
44
45 @sort: AbsolutePathList, ObjectTypeList, RestrictedContentList, RegexMatchList,
46 RegexList, _Vertex, DirectedGraph, PathResolverSingleton,
47 sortDict, convertSize, getUidGid, changeOwnership, splitCommandLine,
48 resolveCommand, executeCommand, calculateFileAge, encodePath, nullDevice,
49 deriveDayOfWeek, isStartOfWeek, buildNormalizedPath,
50 ISO_SECTOR_SIZE, BYTES_PER_SECTOR,
51 BYTES_PER_KBYTE, BYTES_PER_MBYTE, BYTES_PER_GBYTE, KBYTES_PER_MBYTE, MBYTES_PER_GBYTE,
52 SECONDS_PER_MINUTE, MINUTES_PER_HOUR, HOURS_PER_DAY, SECONDS_PER_DAY,
53 UNIT_BYTES, UNIT_KBYTES, UNIT_MBYTES, UNIT_GBYTES, UNIT_SECTORS
54
55 @var ISO_SECTOR_SIZE: Size of an ISO image sector, in bytes.
56 @var BYTES_PER_SECTOR: Number of bytes (B) per ISO sector.
57 @var BYTES_PER_KBYTE: Number of bytes (B) per kilobyte (kB).
58 @var BYTES_PER_MBYTE: Number of bytes (B) per megabyte (MB).
59 @var BYTES_PER_GBYTE: Number of bytes (B) per megabyte (GB).
60 @var KBYTES_PER_MBYTE: Number of kilobytes (kB) per megabyte (MB).
61 @var MBYTES_PER_GBYTE: Number of megabytes (MB) per gigabyte (GB).
62 @var SECONDS_PER_MINUTE: Number of seconds per minute.
63 @var MINUTES_PER_HOUR: Number of minutes per hour.
64 @var HOURS_PER_DAY: Number of hours per day.
65 @var SECONDS_PER_DAY: Number of seconds per day.
66 @var UNIT_BYTES: Constant representing the byte (B) unit for conversion.
67 @var UNIT_KBYTES: Constant representing the kilobyte (kB) unit for conversion.
68 @var UNIT_MBYTES: Constant representing the megabyte (MB) unit for conversion.
69 @var UNIT_GBYTES: Constant representing the gigabyte (GB) unit for conversion.
70 @var UNIT_SECTORS: Constant representing the ISO sector unit for conversion.
71
72 @author: Kenneth J. Pronovici <pronovic@ieee.org>
73 """
74
75
76
77
78
79
80 import sys
81 import math
82 import os
83 import re
84 import time
85 import logging
86 import string
87 from subprocess import Popen, STDOUT, PIPE
88
89 from CedarBackup2.release import VERSION, DATE
90
91 try:
92 import pwd
93 import grp
94 _UID_GID_AVAILABLE = True
95 except ImportError:
96 _UID_GID_AVAILABLE = False
97
98
99
100
101
102
103 logger = logging.getLogger("CedarBackup2.log.util")
104 outputLogger = logging.getLogger("CedarBackup2.output")
105
106 ISO_SECTOR_SIZE = 2048.0
107 BYTES_PER_SECTOR = ISO_SECTOR_SIZE
108
109 BYTES_PER_KBYTE = 1024.0
110 KBYTES_PER_MBYTE = 1024.0
111 MBYTES_PER_GBYTE = 1024.0
112 BYTES_PER_MBYTE = BYTES_PER_KBYTE * KBYTES_PER_MBYTE
113 BYTES_PER_GBYTE = BYTES_PER_MBYTE * MBYTES_PER_GBYTE
114
115 SECONDS_PER_MINUTE = 60.0
116 MINUTES_PER_HOUR = 60.0
117 HOURS_PER_DAY = 24.0
118 SECONDS_PER_DAY = SECONDS_PER_MINUTE * MINUTES_PER_HOUR * HOURS_PER_DAY
119
120 UNIT_BYTES = 0
121 UNIT_KBYTES = 1
122 UNIT_MBYTES = 2
123 UNIT_GBYTES = 4
124 UNIT_SECTORS = 3
125
126 MTAB_FILE = "/etc/mtab"
127
128 MOUNT_COMMAND = [ "mount", ]
129 UMOUNT_COMMAND = [ "umount", ]
130
131 DEFAULT_LANGUAGE = "C"
132 LANG_VAR = "LANG"
133 LOCALE_VARS = [ "LC_ADDRESS", "LC_ALL", "LC_COLLATE",
134 "LC_CTYPE", "LC_IDENTIFICATION",
135 "LC_MEASUREMENT", "LC_MESSAGES",
136 "LC_MONETARY", "LC_NAME", "LC_NUMERIC",
137 "LC_PAPER", "LC_TELEPHONE", "LC_TIME", ]
145
146 """
147 Class representing an "unordered list".
148
149 An "unordered list" is a list in which only the contents matter, not the
150 order in which the contents appear in the list.
151
152 For instance, we might be keeping track of set of paths in a list, because
153 it's convenient to have them in that form. However, for comparison
154 purposes, we would only care that the lists contain exactly the same
155 contents, regardless of order.
156
157 I have come up with two reasonable ways of doing this, plus a couple more
158 that would work but would be a pain to implement. My first method is to
159 copy and sort each list, comparing the sorted versions. This will only work
160 if two lists with exactly the same members are guaranteed to sort in exactly
161 the same order. The second way would be to create two Sets and then compare
162 the sets. However, this would lose information about any duplicates in
163 either list. I've decided to go with option #1 for now. I'll modify this
164 code if I run into problems in the future.
165
166 We override the original C{__eq__}, C{__ne__}, C{__ge__}, C{__gt__},
167 C{__le__} and C{__lt__} list methods to change the definition of the various
168 comparison operators. In all cases, the comparison is changed to return the
169 result of the original operation I{but instead comparing sorted lists}.
170 This is going to be quite a bit slower than a normal list, so you probably
171 only want to use it on small lists.
172 """
173
175 """
176 Definition of C{==} operator for this class.
177 @param other: Other object to compare to.
178 @return: True/false depending on whether C{self == other}.
179 """
180 if other is None:
181 return False
182 selfSorted = self[:]
183 otherSorted = other[:]
184 selfSorted.sort()
185 otherSorted.sort()
186 return selfSorted.__eq__(otherSorted)
187
189 """
190 Definition of C{!=} operator for this class.
191 @param other: Other object to compare to.
192 @return: True/false depending on whether C{self != other}.
193 """
194 if other is None:
195 return True
196 selfSorted = self[:]
197 otherSorted = other[:]
198 selfSorted.sort()
199 otherSorted.sort()
200 return selfSorted.__ne__(otherSorted)
201
203 """
204 Definition of S{>=} operator for this class.
205 @param other: Other object to compare to.
206 @return: True/false depending on whether C{self >= other}.
207 """
208 if other is None:
209 return True
210 selfSorted = self[:]
211 otherSorted = other[:]
212 selfSorted.sort()
213 otherSorted.sort()
214 return selfSorted.__ge__(otherSorted)
215
217 """
218 Definition of C{>} operator for this class.
219 @param other: Other object to compare to.
220 @return: True/false depending on whether C{self > other}.
221 """
222 if other is None:
223 return True
224 selfSorted = self[:]
225 otherSorted = other[:]
226 selfSorted.sort()
227 otherSorted.sort()
228 return selfSorted.__gt__(otherSorted)
229
231 """
232 Definition of S{<=} operator for this class.
233 @param other: Other object to compare to.
234 @return: True/false depending on whether C{self <= other}.
235 """
236 if other is None:
237 return False
238 selfSorted = self[:]
239 otherSorted = other[:]
240 selfSorted.sort()
241 otherSorted.sort()
242 return selfSorted.__le__(otherSorted)
243
245 """
246 Definition of C{<} operator for this class.
247 @param other: Other object to compare to.
248 @return: True/false depending on whether C{self < other}.
249 """
250 if other is None:
251 return False
252 selfSorted = self[:]
253 otherSorted = other[:]
254 selfSorted.sort()
255 otherSorted.sort()
256 return selfSorted.__lt__(otherSorted)
257
264
265 """
266 Class representing a list of absolute paths.
267
268 This is an unordered list.
269
270 We override the C{append}, C{insert} and C{extend} methods to ensure that
271 any item added to the list is an absolute path.
272
273 Each item added to the list is encoded using L{encodePath}. If we don't do
274 this, we have problems trying certain operations between strings and unicode
275 objects, particularly for "odd" filenames that can't be encoded in standard
276 ASCII.
277 """
278
280 """
281 Overrides the standard C{append} method.
282 @raise ValueError: If item is not an absolute path.
283 """
284 if not os.path.isabs(item):
285 raise ValueError("Not an absolute path: [%s]" % item)
286 list.append(self, encodePath(item))
287
288 - def insert(self, index, item):
289 """
290 Overrides the standard C{insert} method.
291 @raise ValueError: If item is not an absolute path.
292 """
293 if not os.path.isabs(item):
294 raise ValueError("Not an absolute path: [%s]" % item)
295 list.insert(self, index, encodePath(item))
296
298 """
299 Overrides the standard C{insert} method.
300 @raise ValueError: If any item is not an absolute path.
301 """
302 for item in seq:
303 if not os.path.isabs(item):
304 raise ValueError("Not an absolute path: [%s]" % item)
305 for item in seq:
306 list.append(self, encodePath(item))
307
314
315 """
316 Class representing a list containing only objects with a certain type.
317
318 This is an unordered list.
319
320 We override the C{append}, C{insert} and C{extend} methods to ensure that
321 any item added to the list matches the type that is requested. The
322 comparison uses the built-in C{isinstance}, which should allow subclasses of
323 of the requested type to be added to the list as well.
324
325 The C{objectName} value will be used in exceptions, i.e. C{"Item must be a
326 CollectDir object."} if C{objectName} is C{"CollectDir"}.
327 """
328
329 - def __init__(self, objectType, objectName):
330 """
331 Initializes a typed list for a particular type.
332 @param objectType: Type that the list elements must match.
333 @param objectName: Short string containing the "name" of the type.
334 """
335 super(ObjectTypeList, self).__init__()
336 self.objectType = objectType
337 self.objectName = objectName
338
340 """
341 Overrides the standard C{append} method.
342 @raise ValueError: If item does not match requested type.
343 """
344 if not isinstance(item, self.objectType):
345 raise ValueError("Item must be a %s object." % self.objectName)
346 list.append(self, item)
347
348 - def insert(self, index, item):
349 """
350 Overrides the standard C{insert} method.
351 @raise ValueError: If item does not match requested type.
352 """
353 if not isinstance(item, self.objectType):
354 raise ValueError("Item must be a %s object." % self.objectName)
355 list.insert(self, index, item)
356
358 """
359 Overrides the standard C{insert} method.
360 @raise ValueError: If item does not match requested type.
361 """
362 for item in seq:
363 if not isinstance(item, self.objectType):
364 raise ValueError("All items must be %s objects." % self.objectName)
365 list.extend(self, seq)
366
367
368
369
370
371
372 -class RestrictedContentList(UnorderedList):
373
374 """
375 Class representing a list containing only object with certain values.
376
377 This is an unordered list.
378
379 We override the C{append}, C{insert} and C{extend} methods to ensure that
380 any item added to the list is among the valid values. We use a standard
381 comparison, so pretty much anything can be in the list of valid values.
382
383 The C{valuesDescr} value will be used in exceptions, i.e. C{"Item must be
384 one of values in VALID_ACTIONS"} if C{valuesDescr} is C{"VALID_ACTIONS"}.
385
386 @note: This class doesn't make any attempt to trap for nonsensical
387 arguments. All of the values in the values list should be of the same type
388 (i.e. strings). Then, all list operations also need to be of that type
389 (i.e. you should always insert or append just strings). If you mix types --
390 for instance lists and strings -- you will likely see AttributeError
391 exceptions or other problems.
392 """
393
394 - def __init__(self, valuesList, valuesDescr, prefix=None):
395 """
396 Initializes a list restricted to containing certain values.
397 @param valuesList: List of valid values.
398 @param valuesDescr: Short string describing list of values.
399 @param prefix: Prefix to use in error messages (None results in prefix "Item")
400 """
401 super(RestrictedContentList, self).__init__()
402 self.prefix = "Item"
403 if prefix is not None: self.prefix = prefix
404 self.valuesList = valuesList
405 self.valuesDescr = valuesDescr
406
407 - def append(self, item):
408 """
409 Overrides the standard C{append} method.
410 @raise ValueError: If item is not in the values list.
411 """
412 if item not in self.valuesList:
413 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr))
414 list.append(self, item)
415
416 - def insert(self, index, item):
417 """
418 Overrides the standard C{insert} method.
419 @raise ValueError: If item is not in the values list.
420 """
421 if item not in self.valuesList:
422 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr))
423 list.insert(self, index, item)
424
425 - def extend(self, seq):
426 """
427 Overrides the standard C{insert} method.
428 @raise ValueError: If item is not in the values list.
429 """
430 for item in seq:
431 if item not in self.valuesList:
432 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr))
433 list.extend(self, seq)
434
441
442 """
443 Class representing a list containing only strings that match a regular expression.
444
445 If C{emptyAllowed} is passed in as C{False}, then empty strings are
446 explicitly disallowed, even if they happen to match the regular expression.
447 (C{None} values are always disallowed, since string operations are not
448 permitted on C{None}.)
449
450 This is an unordered list.
451
452 We override the C{append}, C{insert} and C{extend} methods to ensure that
453 any item added to the list matches the indicated regular expression.
454
455 @note: If you try to put values that are not strings into the list, you will
456 likely get either TypeError or AttributeError exceptions as a result.
457 """
458
459 - def __init__(self, valuesRegex, emptyAllowed=True, prefix=None):
460 """
461 Initializes a list restricted to containing certain values.
462 @param valuesRegex: Regular expression that must be matched, as a string
463 @param emptyAllowed: Indicates whether empty or None values are allowed.
464 @param prefix: Prefix to use in error messages (None results in prefix "Item")
465 """
466 super(RegexMatchList, self).__init__()
467 self.prefix = "Item"
468 if prefix is not None: self.prefix = prefix
469 self.valuesRegex = valuesRegex
470 self.emptyAllowed = emptyAllowed
471 self.pattern = re.compile(self.valuesRegex)
472
474 """
475 Overrides the standard C{append} method.
476 @raise ValueError: If item is None
477 @raise ValueError: If item is empty and empty values are not allowed
478 @raise ValueError: If item does not match the configured regular expression
479 """
480 if item is None or (not self.emptyAllowed and item == ""):
481 raise ValueError("%s cannot be empty." % self.prefix)
482 if not self.pattern.search(item):
483 raise ValueError("%s is not valid: [%s]" % (self.prefix, item))
484 list.append(self, item)
485
486 - def insert(self, index, item):
487 """
488 Overrides the standard C{insert} method.
489 @raise ValueError: If item is None
490 @raise ValueError: If item is empty and empty values are not allowed
491 @raise ValueError: If item does not match the configured regular expression
492 """
493 if item is None or (not self.emptyAllowed and item == ""):
494 raise ValueError("%s cannot be empty." % self.prefix)
495 if not self.pattern.search(item):
496 raise ValueError("%s is not valid [%s]" % (self.prefix, item))
497 list.insert(self, index, item)
498
500 """
501 Overrides the standard C{insert} method.
502 @raise ValueError: If any item is None
503 @raise ValueError: If any item is empty and empty values are not allowed
504 @raise ValueError: If any item does not match the configured regular expression
505 """
506 for item in seq:
507 if item is None or (not self.emptyAllowed and item == ""):
508 raise ValueError("%s cannot be empty.", self.prefix)
509 if not self.pattern.search(item):
510 raise ValueError("%s is not valid: [%s]" % (self.prefix, item))
511 list.extend(self, seq)
512
513
514
515
516
517
518 -class RegexList(UnorderedList):
519
520 """
521 Class representing a list of valid regular expression strings.
522
523 This is an unordered list.
524
525 We override the C{append}, C{insert} and C{extend} methods to ensure that
526 any item added to the list is a valid regular expression.
527 """
528
530 """
531 Overrides the standard C{append} method.
532 @raise ValueError: If item is not an absolute path.
533 """
534 try:
535 re.compile(item)
536 except re.error:
537 raise ValueError("Not a valid regular expression: [%s]" % item)
538 list.append(self, item)
539
540 - def insert(self, index, item):
541 """
542 Overrides the standard C{insert} method.
543 @raise ValueError: If item is not an absolute path.
544 """
545 try:
546 re.compile(item)
547 except re.error:
548 raise ValueError("Not a valid regular expression: [%s]" % item)
549 list.insert(self, index, item)
550
552 """
553 Overrides the standard C{insert} method.
554 @raise ValueError: If any item is not an absolute path.
555 """
556 for item in seq:
557 try:
558 re.compile(item)
559 except re.error:
560 raise ValueError("Not a valid regular expression: [%s]" % item)
561 for item in seq:
562 list.append(self, item)
563
564
565
566
567
568
569 -class _Vertex(object):
570
571 """
572 Represents a vertex (or node) in a directed graph.
573 """
574
576 """
577 Constructor.
578 @param name: Name of this graph vertex.
579 @type name: String value.
580 """
581 self.name = name
582 self.endpoints = []
583 self.state = None
584
586
587 """
588 Represents a directed graph.
589
590 A graph B{G=(V,E)} consists of a set of vertices B{V} together with a set
591 B{E} of vertex pairs or edges. In a directed graph, each edge also has an
592 associated direction (from vertext B{v1} to vertex B{v2}). A C{DirectedGraph}
593 object provides a way to construct a directed graph and execute a depth-
594 first search.
595
596 This data structure was designed based on the graphing chapter in
597 U{The Algorithm Design Manual<http://www2.toki.or.id/book/AlgDesignManual/>},
598 by Steven S. Skiena.
599
600 This class is intended to be used by Cedar Backup for dependency ordering.
601 Because of this, it's not quite general-purpose. Unlike a "general" graph,
602 every vertex in this graph has at least one edge pointing to it, from a
603 special "start" vertex. This is so no vertices get "lost" either because
604 they have no dependencies or because nothing depends on them.
605 """
606
607 _UNDISCOVERED = 0
608 _DISCOVERED = 1
609 _EXPLORED = 2
610
612 """
613 Directed graph constructor.
614
615 @param name: Name of this graph.
616 @type name: String value.
617 """
618 if name is None or name == "":
619 raise ValueError("Graph name must be non-empty.")
620 self._name = name
621 self._vertices = {}
622 self._startVertex = _Vertex(None)
623
625 """
626 Official string representation for class instance.
627 """
628 return "DirectedGraph(%s)" % self.name
629
631 """
632 Informal string representation for class instance.
633 """
634 return self.__repr__()
635
637 """
638 Definition of equals operator for this class.
639 @param other: Other object to compare to.
640 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
641 """
642
643 if other is None:
644 return 1
645 if self.name != other.name:
646 if self.name < other.name:
647 return -1
648 else:
649 return 1
650 if self._vertices != other._vertices:
651 if self._vertices < other._vertices:
652 return -1
653 else:
654 return 1
655 return 0
656
658 """
659 Property target used to get the graph name.
660 """
661 return self._name
662
663 name = property(_getName, None, None, "Name of the graph.")
664
666 """
667 Creates a named vertex.
668 @param name: vertex name
669 @raise ValueError: If the vertex name is C{None} or empty.
670 """
671 if name is None or name == "":
672 raise ValueError("Vertex name must be non-empty.")
673 vertex = _Vertex(name)
674 self._startVertex.endpoints.append(vertex)
675 self._vertices[name] = vertex
676
678 """
679 Adds an edge with an associated direction, from C{start} vertex to C{finish} vertex.
680 @param start: Name of start vertex.
681 @param finish: Name of finish vertex.
682 @raise ValueError: If one of the named vertices is unknown.
683 """
684 try:
685 startVertex = self._vertices[start]
686 finishVertex = self._vertices[finish]
687 startVertex.endpoints.append(finishVertex)
688 except KeyError, e:
689 raise ValueError("Vertex [%s] could not be found." % e)
690
692 """
693 Implements a topological sort of the graph.
694
695 This method also enforces that the graph is a directed acyclic graph,
696 which is a requirement of a topological sort.
697
698 A directed acyclic graph (or "DAG") is a directed graph with no directed
699 cycles. A topological sort of a DAG is an ordering on the vertices such
700 that all edges go from left to right. Only an acyclic graph can have a
701 topological sort, but any DAG has at least one topological sort.
702
703 Since a topological sort only makes sense for an acyclic graph, this
704 method throws an exception if a cycle is found.
705
706 A depth-first search only makes sense if the graph is acyclic. If the
707 graph contains any cycles, it is not possible to determine a consistent
708 ordering for the vertices.
709
710 @note: If a particular vertex has no edges, then its position in the
711 final list depends on the order in which the vertices were created in the
712 graph. If you're using this method to determine a dependency order, this
713 makes sense: a vertex with no dependencies can go anywhere (and will).
714
715 @return: Ordering on the vertices so that all edges go from left to right.
716
717 @raise ValueError: If a cycle is found in the graph.
718 """
719 ordering = []
720 for key in self._vertices:
721 vertex = self._vertices[key]
722 vertex.state = self._UNDISCOVERED
723 for key in self._vertices:
724 vertex = self._vertices[key]
725 if vertex.state == self._UNDISCOVERED:
726 self._topologicalSort(self._startVertex, ordering)
727 return ordering
728
730 """
731 Recursive depth first search function implementing topological sort.
732 @param vertex: Vertex to search
733 @param ordering: List of vertices in proper order
734 """
735 vertex.state = self._DISCOVERED
736 for endpoint in vertex.endpoints:
737 if endpoint.state == self._UNDISCOVERED:
738 self._topologicalSort(endpoint, ordering)
739 elif endpoint.state != self._EXPLORED:
740 raise ValueError("Cycle found in graph (found '%s' while searching '%s')." % (vertex.name, endpoint.name))
741 if vertex.name is not None:
742 ordering.insert(0, vertex.name)
743 vertex.state = self._EXPLORED
744
751
752 """
753 Singleton used for resolving executable paths.
754
755 Various functions throughout Cedar Backup (including extensions) need a way
756 to resolve the path of executables that they use. For instance, the image
757 functionality needs to find the C{mkisofs} executable, and the Subversion
758 extension needs to find the C{svnlook} executable. Cedar Backup's original
759 behavior was to assume that the simple name (C{"svnlook"} or whatever) was
760 available on the caller's C{$PATH}, and to fail otherwise. However, this
761 turns out to be less than ideal, since for instance the root user might not
762 always have executables like C{svnlook} in its path.
763
764 One solution is to specify a path (either via an absolute path or some sort
765 of path insertion or path appending mechanism) that would apply to the
766 C{executeCommand()} function. This is not difficult to implement, but it
767 seem like kind of a "big hammer" solution. Besides that, it might also
768 represent a security flaw (for instance, I prefer not to mess with root's
769 C{$PATH} on the application level if I don't have to).
770
771 The alternative is to set up some sort of configuration for the path to
772 certain executables, i.e. "find C{svnlook} in C{/usr/local/bin/svnlook}" or
773 whatever. This PathResolverSingleton aims to provide a good solution to the
774 mapping problem. Callers of all sorts (extensions or not) can get an
775 instance of the singleton. Then, they call the C{lookup} method to try and
776 resolve the executable they are looking for. Through the C{lookup} method,
777 the caller can also specify a default to use if a mapping is not found.
778 This way, with no real effort on the part of the caller, behavior can neatly
779 degrade to something equivalent to the current behavior if there is no
780 special mapping or if the singleton was never initialized in the first
781 place.
782
783 Even better, extensions automagically get access to the same resolver
784 functionality, and they don't even need to understand how the mapping
785 happens. All extension authors need to do is document what executables
786 their code requires, and the standard resolver configuration section will
787 meet their needs.
788
789 The class should be initialized once through the constructor somewhere in
790 the main routine. Then, the main routine should call the L{fill} method to
791 fill in the resolver's internal structures. Everyone else who needs to
792 resolve a path will get an instance of the class using L{getInstance} and
793 will then just call the L{lookup} method.
794
795 @cvar _instance: Holds a reference to the singleton
796 @ivar _mapping: Internal mapping from resource name to path.
797 """
798
799 _instance = None
800
802 """Helper class to provide a singleton factory method."""
811
812 getInstance = _Helper()
813
820
821 - def lookup(self, name, default=None):
822 """
823 Looks up name and returns the resolved path associated with the name.
824 @param name: Name of the path resource to resolve.
825 @param default: Default to return if resource cannot be resolved.
826 @return: Resolved path associated with name, or default if name can't be resolved.
827 """
828 value = default
829 if name in self._mapping.keys():
830 value = self._mapping[name]
831 logger.debug("Resolved command [%s] to [%s]." % (name, value))
832 return value
833
834 - def fill(self, mapping):
835 """
836 Fills in the singleton's internal mapping from name to resource.
837 @param mapping: Mapping from resource name to path.
838 @type mapping: Dictionary mapping name to path, both as strings.
839 """
840 self._mapping = { }
841 for key in mapping.keys():
842 self._mapping[key] = mapping[key]
843
844
845
846
847
848
849 -class Pipe(Popen):
850 """
851 Specialized pipe class for use by C{executeCommand}.
852
853 The L{executeCommand} function needs a specialized way of interacting
854 with a pipe. First, C{executeCommand} only reads from the pipe, and
855 never writes to it. Second, C{executeCommand} needs a way to discard all
856 output written to C{stderr}, as a means of simulating the shell
857 C{2>/dev/null} construct.
858 """
859 - def __init__(self, cmd, bufsize=-1, ignoreStderr=False):
860 stderr = STDOUT
861 if ignoreStderr:
862 devnull = nullDevice()
863 stderr = os.open(devnull, os.O_RDWR)
864 Popen.__init__(self, shell=False, args=cmd, bufsize=bufsize, stdin=None, stdout=PIPE, stderr=stderr)
865
872
873 """
874 Class holding runtime diagnostic information.
875
876 Diagnostic information is information that is useful to get from users for
877 debugging purposes. I'm consolidating it all here into one object.
878
879 @sort: __init__, __repr__, __str__
880 """
881
882
884 """
885 Constructor for the C{Diagnostics} class.
886 """
887
889 """
890 Official string representation for class instance.
891 """
892 return "Diagnostics()"
893
895 """
896 Informal string representation for class instance.
897 """
898 return self.__repr__()
899
901 """
902 Get a map containing all of the diagnostic values.
903 @return: Map from diagnostic name to diagnostic value.
904 """
905 values = {}
906 values['version'] = self.version
907 values['interpreter'] = self.interpreter
908 values['platform'] = self.platform
909 values['encoding'] = self.encoding
910 values['locale'] = self.locale
911 values['timestamp'] = self.timestamp
912 return values
913
915 """
916 Pretty-print diagnostic information to a file descriptor.
917 @param fd: File descriptor used to print information.
918 @param prefix: Prefix string (if any) to place onto printed lines
919 @note: The C{fd} is used rather than C{print} to facilitate unit testing.
920 """
921 lines = self._buildDiagnosticLines(prefix)
922 for line in lines:
923 fd.write("%s\n" % line)
924
926 """
927 Pretty-print diagnostic information using a logger method.
928 @param method: Logger method to use for logging (i.e. logger.info)
929 @param prefix: Prefix string (if any) to place onto printed lines
930 """
931 lines = self._buildDiagnosticLines(prefix)
932 for line in lines:
933 method("%s" % line)
934
936 """
937 Build a set of pretty-printed diagnostic lines.
938 @param prefix: Prefix string (if any) to place onto printed lines
939 @return: List of strings, not terminated by newlines.
940 """
941 values = self.getValues()
942 keys = values.keys()
943 keys.sort()
944 tmax = Diagnostics._getMaxLength(keys) + 3
945 lines = []
946 for key in keys:
947 title = key.title()
948 title += (tmax - len(title)) * '.'
949 value = values[key]
950 line = "%s%s: %s" % (prefix, title, value)
951 lines.append(line)
952 return lines
953
954 @staticmethod
956 """
957 Get the maximum length from among a list of strings.
958 """
959 tmax = 0
960 for value in values:
961 if len(value) > tmax:
962 tmax = len(value)
963 return tmax
964
966 """
967 Property target to get the Cedar Backup version.
968 """
969 return "Cedar Backup %s (%s)" % (VERSION, DATE)
970
972 """
973 Property target to get the Python interpreter version.
974 """
975 version = sys.version_info
976 return "Python %d.%d.%d (%s)" % (version[0], version[1], version[2], version[3])
977
979 """
980 Property target to get the filesystem encoding.
981 """
982 return sys.getfilesystemencoding() or sys.getdefaultencoding()
983
1004
1006 """
1007 Property target to get the default locale that is in effect.
1008 """
1009 try:
1010 import locale
1011 return locale.getdefaultlocale()[0]
1012 except:
1013 return "(unknown)"
1014
1016 """
1017 Property target to get a current date/time stamp.
1018 """
1019 try:
1020 import datetime
1021 return datetime.datetime.utcnow().ctime() + " UTC"
1022 except:
1023 return "(unknown)"
1024
1025 version = property(_getVersion, None, None, "Cedar Backup version.")
1026 interpreter = property(_getInterpreter, None, None, "Python interpreter version.")
1027 platform = property(_getPlatform, None, None, "Platform identifying information.")
1028 encoding = property(_getEncoding, None, None, "Filesystem encoding that is in effect.")
1029 locale = property(_getLocale, None, None, "Locale that is in effect.")
1030 timestamp = property(_getTimestamp, None, None, "Current timestamp.")
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041 -def sortDict(d):
1042 """
1043 Returns the keys of the dictionary sorted by value.
1044
1045 There are cuter ways to do this in Python 2.4, but we were originally
1046 attempting to stay compatible with Python 2.3.
1047
1048 @param d: Dictionary to operate on
1049 @return: List of dictionary keys sorted in order by dictionary value.
1050 """
1051 items = d.items()
1052 items.sort(lambda x, y: cmp(x[1], y[1]))
1053 return [key for key, value in items]
1054
1055
1056
1057
1058
1059
1060 -def removeKeys(d, keys):
1061 """
1062 Removes all of the keys from the dictionary.
1063 The dictionary is altered in-place.
1064 Each key must exist in the dictionary.
1065 @param d: Dictionary to operate on
1066 @param keys: List of keys to remove
1067 @raise KeyError: If one of the keys does not exist
1068 """
1069 for key in keys:
1070 del d[key]
1071
1072
1073
1074
1075
1076
1077 -def convertSize(size, fromUnit, toUnit):
1078 """
1079 Converts a size in one unit to a size in another unit.
1080
1081 This is just a convenience function so that the functionality can be
1082 implemented in just one place. Internally, we convert values to bytes and
1083 then to the final unit.
1084
1085 The available units are:
1086
1087 - C{UNIT_BYTES} - Bytes
1088 - C{UNIT_KBYTES} - Kilobytes, where 1 kB = 1024 B
1089 - C{UNIT_MBYTES} - Megabytes, where 1 MB = 1024 kB
1090 - C{UNIT_GBYTES} - Gigabytes, where 1 GB = 1024 MB
1091 - C{UNIT_SECTORS} - Sectors, where 1 sector = 2048 B
1092
1093 @param size: Size to convert
1094 @type size: Integer or float value in units of C{fromUnit}
1095
1096 @param fromUnit: Unit to convert from
1097 @type fromUnit: One of the units listed above
1098
1099 @param toUnit: Unit to convert to
1100 @type toUnit: One of the units listed above
1101
1102 @return: Number converted to new unit, as a float.
1103 @raise ValueError: If one of the units is invalid.
1104 """
1105 if size is None:
1106 raise ValueError("Cannot convert size of None.")
1107 if fromUnit == UNIT_BYTES:
1108 byteSize = float(size)
1109 elif fromUnit == UNIT_KBYTES:
1110 byteSize = float(size) * BYTES_PER_KBYTE
1111 elif fromUnit == UNIT_MBYTES:
1112 byteSize = float(size) * BYTES_PER_MBYTE
1113 elif fromUnit == UNIT_GBYTES:
1114 byteSize = float(size) * BYTES_PER_GBYTE
1115 elif fromUnit == UNIT_SECTORS:
1116 byteSize = float(size) * BYTES_PER_SECTOR
1117 else:
1118 raise ValueError("Unknown 'from' unit %s." % fromUnit)
1119 if toUnit == UNIT_BYTES:
1120 return byteSize
1121 elif toUnit == UNIT_KBYTES:
1122 return byteSize / BYTES_PER_KBYTE
1123 elif toUnit == UNIT_MBYTES:
1124 return byteSize / BYTES_PER_MBYTE
1125 elif toUnit == UNIT_GBYTES:
1126 return byteSize / BYTES_PER_GBYTE
1127 elif toUnit == UNIT_SECTORS:
1128 return byteSize / BYTES_PER_SECTOR
1129 else:
1130 raise ValueError("Unknown 'to' unit %s." % toUnit)
1131
1132
1133
1134
1135
1136
1137 -def displayBytes(bytes, digits=2):
1138 """
1139 Format a byte quantity so it can be sensibly displayed.
1140
1141 It's rather difficult to look at a number like "72372224 bytes" and get any
1142 meaningful information out of it. It would be more useful to see something
1143 like "69.02 MB". That's what this function does. Any time you want to display
1144 a byte value, i.e.::
1145
1146 print "Size: %s bytes" % bytes
1147
1148 Call this function instead::
1149
1150 print "Size: %s" % displayBytes(bytes)
1151
1152 What comes out will be sensibly formatted. The indicated number of digits
1153 will be listed after the decimal point, rounded based on whatever rules are
1154 used by Python's standard C{%f} string format specifier. (Values less than 1
1155 kB will be listed in bytes and will not have a decimal point, since the
1156 concept of a fractional byte is nonsensical.)
1157
1158 @param bytes: Byte quantity.
1159 @type bytes: Integer number of bytes.
1160
1161 @param digits: Number of digits to display after the decimal point.
1162 @type digits: Integer value, typically 2-5.
1163
1164 @return: String, formatted for sensible display.
1165 """
1166 if(bytes is None):
1167 raise ValueError("Cannot display byte value of None.")
1168 bytes = float(bytes)
1169 if math.fabs(bytes) < BYTES_PER_KBYTE:
1170 fmt = "%.0f bytes"
1171 value = bytes
1172 elif math.fabs(bytes) < BYTES_PER_MBYTE:
1173 fmt = "%." + "%d" % digits + "f kB"
1174 value = bytes / BYTES_PER_KBYTE
1175 elif math.fabs(bytes) < BYTES_PER_GBYTE:
1176 fmt = "%." + "%d" % digits + "f MB"
1177 value = bytes / BYTES_PER_MBYTE
1178 else:
1179 fmt = "%." + "%d" % digits + "f GB"
1180 value = bytes / BYTES_PER_GBYTE
1181 return fmt % value
1182
1189 """
1190 Gets a reference to a named function.
1191
1192 This does some hokey-pokey to get back a reference to a dynamically named
1193 function. For instance, say you wanted to get a reference to the
1194 C{os.path.isdir} function. You could use::
1195
1196 myfunc = getFunctionReference("os.path", "isdir")
1197
1198 Although we won't bomb out directly, behavior is pretty much undefined if
1199 you pass in C{None} or C{""} for either C{module} or C{function}.
1200
1201 The only validation we enforce is that whatever we get back must be
1202 callable.
1203
1204 I derived this code based on the internals of the Python unittest
1205 implementation. I don't claim to completely understand how it works.
1206
1207 @param module: Name of module associated with function.
1208 @type module: Something like "os.path" or "CedarBackup2.util"
1209
1210 @param function: Name of function
1211 @type function: Something like "isdir" or "getUidGid"
1212
1213 @return: Reference to function associated with name.
1214
1215 @raise ImportError: If the function cannot be found.
1216 @raise ValueError: If the resulting reference is not callable.
1217
1218 @copyright: Some of this code, prior to customization, was originally part
1219 of the Python 2.3 codebase. Python code is copyright (c) 2001, 2002 Python
1220 Software Foundation; All Rights Reserved.
1221 """
1222 parts = []
1223 if module is not None and module != "":
1224 parts = module.split(".")
1225 if function is not None and function != "":
1226 parts.append(function)
1227 copy = parts[:]
1228 while copy:
1229 try:
1230 module = __import__(string.join(copy, "."))
1231 break
1232 except ImportError:
1233 del copy[-1]
1234 if not copy: raise
1235 parts = parts[1:]
1236 obj = module
1237 for part in parts:
1238 obj = getattr(obj, part)
1239 if not callable(obj):
1240 raise ValueError("Reference to %s.%s is not callable." % (module, function))
1241 return obj
1242
1243
1244
1245
1246
1247
1248 -def getUidGid(user, group):
1249 """
1250 Get the uid/gid associated with a user/group pair
1251
1252 This is a no-op if user/group functionality is not available on the platform.
1253
1254 @param user: User name
1255 @type user: User name as a string
1256
1257 @param group: Group name
1258 @type group: Group name as a string
1259
1260 @return: Tuple C{(uid, gid)} matching passed-in user and group.
1261 @raise ValueError: If the ownership user/group values are invalid
1262 """
1263 if _UID_GID_AVAILABLE:
1264 try:
1265 uid = pwd.getpwnam(user)[2]
1266 gid = grp.getgrnam(group)[2]
1267 return (uid, gid)
1268 except Exception, e:
1269 logger.debug("Error looking up uid and gid for [%s:%s]: %s" % (user, group, e))
1270 raise ValueError("Unable to lookup up uid and gid for passed in user/group.")
1271 else:
1272 return (0, 0)
1273
1280 """
1281 Changes ownership of path to match the user and group.
1282
1283 This is a no-op if user/group functionality is not available on the
1284 platform, or if the either passed-in user or group is C{None}. Further, we
1285 won't even try to do it unless running as root, since it's unlikely to work.
1286
1287 @param path: Path whose ownership to change.
1288 @param user: User which owns file.
1289 @param group: Group which owns file.
1290 """
1291 if _UID_GID_AVAILABLE:
1292 if user is None or group is None:
1293 logger.debug("User or group is None, so not attempting to change owner on [%s]." % path)
1294 elif not isRunningAsRoot():
1295 logger.debug("Not root, so not attempting to change owner on [%s]." % path)
1296 else:
1297 try:
1298 (uid, gid) = getUidGid(user, group)
1299 os.chown(path, uid, gid)
1300 except Exception, e:
1301 logger.error("Error changing ownership of [%s]: %s" % (path, e))
1302
1309 """
1310 Indicates whether the program is running as the root user.
1311 """
1312 return os.getuid() == 0
1313
1320 """
1321 Splits a command line string into a list of arguments.
1322
1323 Unfortunately, there is no "standard" way to parse a command line string,
1324 and it's actually not an easy problem to solve portably (essentially, we
1325 have to emulate the shell argument-processing logic). This code only
1326 respects double quotes (C{"}) for grouping arguments, not single quotes
1327 (C{'}). Make sure you take this into account when building your command
1328 line.
1329
1330 Incidentally, I found this particular parsing method while digging around in
1331 Google Groups, and I tweaked it for my own use.
1332
1333 @param commandLine: Command line string
1334 @type commandLine: String, i.e. "cback --verbose stage store"
1335
1336 @return: List of arguments, suitable for passing to C{popen2}.
1337
1338 @raise ValueError: If the command line is None.
1339 """
1340 if commandLine is None:
1341 raise ValueError("Cannot split command line of None.")
1342 fields = re.findall('[^ "]+|"[^"]+"', commandLine)
1343 fields = map(lambda field: field.replace('"', ''), fields)
1344 return fields
1345
1352 """
1353 Resolves the real path to a command through the path resolver mechanism.
1354
1355 Both extensions and standard Cedar Backup functionality need a way to
1356 resolve the "real" location of various executables. Normally, they assume
1357 that these executables are on the system path, but some callers need to
1358 specify an alternate location.
1359
1360 Ideally, we want to handle this configuration in a central location. The
1361 Cedar Backup path resolver mechanism (a singleton called
1362 L{PathResolverSingleton}) provides the central location to store the
1363 mappings. This function wraps access to the singleton, and is what all
1364 functions (extensions or standard functionality) should call if they need to
1365 find a command.
1366
1367 The passed-in command must actually be a list, in the standard form used by
1368 all existing Cedar Backup code (something like C{["svnlook", ]}). The
1369 lookup will actually be done on the first element in the list, and the
1370 returned command will always be in list form as well.
1371
1372 If the passed-in command can't be resolved or no mapping exists, then the
1373 command itself will be returned unchanged. This way, we neatly fall back on
1374 default behavior if we have no sensible alternative.
1375
1376 @param command: Command to resolve.
1377 @type command: List form of command, i.e. C{["svnlook", ]}.
1378
1379 @return: Path to command or just command itself if no mapping exists.
1380 """
1381 singleton = PathResolverSingleton.getInstance()
1382 name = command[0]
1383 result = command[:]
1384 result[0] = singleton.lookup(name, name)
1385 return result
1386
1387
1388
1389
1390
1391
1392 -def executeCommand(command, args, returnOutput=False, ignoreStderr=False, doNotLog=False, outputFile=None):
1393 """
1394 Executes a shell command, hopefully in a safe way.
1395
1396 This function exists to replace direct calls to C{os.popen} in the Cedar
1397 Backup code. It's not safe to call a function such as C{os.popen()} with
1398 untrusted arguments, since that can cause problems if the string contains
1399 non-safe variables or other constructs (imagine that the argument is
1400 C{$WHATEVER}, but C{$WHATEVER} contains something like C{"; rm -fR ~/;
1401 echo"} in the current environment).
1402
1403 Instead, it's safer to pass a list of arguments in the style supported bt
1404 C{popen2} or C{popen4}. This function actually uses a specialized C{Pipe}
1405 class implemented using either C{subprocess.Popen} or C{popen2.Popen4}.
1406
1407 Under the normal case, this function will return a tuple of C{(status,
1408 None)} where the status is the wait-encoded return status of the call per
1409 the C{popen2.Popen4} documentation. If C{returnOutput} is passed in as
1410 C{True}, the function will return a tuple of C{(status, output)} where
1411 C{output} is a list of strings, one entry per line in the output from the
1412 command. Output is always logged to the C{outputLogger.info()} target,
1413 regardless of whether it's returned.
1414
1415 By default, C{stdout} and C{stderr} will be intermingled in the output.
1416 However, if you pass in C{ignoreStderr=True}, then only C{stdout} will be
1417 included in the output.
1418
1419 The C{doNotLog} parameter exists so that callers can force the function to
1420 not log command output to the debug log. Normally, you would want to log.
1421 However, if you're using this function to write huge output files (i.e.
1422 database backups written to C{stdout}) then you might want to avoid putting
1423 all that information into the debug log.
1424
1425 The C{outputFile} parameter exists to make it easier for a caller to push
1426 output into a file, i.e. as a substitute for redirection to a file. If this
1427 value is passed in, each time a line of output is generated, it will be
1428 written to the file using C{outputFile.write()}. At the end, the file
1429 descriptor will be flushed using C{outputFile.flush()}. The caller
1430 maintains responsibility for closing the file object appropriately.
1431
1432 @note: I know that it's a bit confusing that the command and the arguments
1433 are both lists. I could have just required the caller to pass in one big
1434 list. However, I think it makes some sense to keep the command (the
1435 constant part of what we're executing, i.e. C{"scp -B"}) separate from its
1436 arguments, even if they both end up looking kind of similar.
1437
1438 @note: You cannot redirect output via shell constructs (i.e. C{>file},
1439 C{2>/dev/null}, etc.) using this function. The redirection string would be
1440 passed to the command just like any other argument. However, you can
1441 implement the equivalent to redirection using C{ignoreStderr} and
1442 C{outputFile}, as discussed above.
1443
1444 @note: The operating system environment is partially sanitized before
1445 the command is invoked. See L{sanitizeEnvironment} for details.
1446
1447 @param command: Shell command to execute
1448 @type command: List of individual arguments that make up the command
1449
1450 @param args: List of arguments to the command
1451 @type args: List of additional arguments to the command
1452
1453 @param returnOutput: Indicates whether to return the output of the command
1454 @type returnOutput: Boolean C{True} or C{False}
1455
1456 @param ignoreStderr: Whether stderr should be discarded
1457 @type ignoreStderr: Boolean True or False
1458
1459 @param doNotLog: Indicates that output should not be logged.
1460 @type doNotLog: Boolean C{True} or C{False}
1461
1462 @param outputFile: File object that all output should be written to.
1463 @type outputFile: File object as returned from C{open()} or C{file()}.
1464
1465 @return: Tuple of C{(result, output)} as described above.
1466 """
1467 logger.debug("Executing command %s with args %s." % (command, args))
1468 outputLogger.info("Executing command %s with args %s." % (command, args))
1469 if doNotLog:
1470 logger.debug("Note: output will not be logged, per the doNotLog flag.")
1471 outputLogger.info("Note: output will not be logged, per the doNotLog flag.")
1472 output = []
1473 fields = command[:]
1474 fields.extend(args)
1475 try:
1476 sanitizeEnvironment()
1477 try:
1478 pipe = Pipe(fields, ignoreStderr=ignoreStderr)
1479 except OSError:
1480
1481
1482
1483 pipe = Pipe(fields, ignoreStderr=ignoreStderr)
1484 while True:
1485 line = pipe.stdout.readline()
1486 if not line: break
1487 if returnOutput: output.append(line)
1488 if outputFile is not None: outputFile.write(line)
1489 if not doNotLog: outputLogger.info(line[:-1])
1490 if outputFile is not None:
1491 try:
1492 outputFile.flush()
1493 except: pass
1494 if returnOutput:
1495 return (pipe.wait(), output)
1496 else:
1497 return (pipe.wait(), None)
1498 except OSError, e:
1499 try:
1500 if returnOutput:
1501 if output != []:
1502 return (pipe.wait(), output)
1503 else:
1504 return (pipe.wait(), [ e, ])
1505 else:
1506 return (pipe.wait(), None)
1507 except UnboundLocalError:
1508 if returnOutput:
1509 return (256, [])
1510 else:
1511 return (256, None)
1512
1519 """
1520 Calculates the age (in days) of a file.
1521
1522 The "age" of a file is the amount of time since the file was last used, per
1523 the most recent of the file's C{st_atime} and C{st_mtime} values.
1524
1525 Technically, we only intend this function to work with files, but it will
1526 probably work with anything on the filesystem.
1527
1528 @param path: Path to a file on disk.
1529
1530 @return: Age of the file in days (possibly fractional).
1531 @raise OSError: If the file doesn't exist.
1532 """
1533 currentTime = int(time.time())
1534 fileStats = os.stat(path)
1535 lastUse = max(fileStats.st_atime, fileStats.st_mtime)
1536 ageInSeconds = currentTime - lastUse
1537 ageInDays = ageInSeconds / SECONDS_PER_DAY
1538 return ageInDays
1539
1540
1541
1542
1543
1544
1545 -def mount(devicePath, mountPoint, fsType):
1546 """
1547 Mounts the indicated device at the indicated mount point.
1548
1549 For instance, to mount a CD, you might use device path C{/dev/cdrw}, mount
1550 point C{/media/cdrw} and filesystem type C{iso9660}. You can safely use any
1551 filesystem type that is supported by C{mount} on your platform. If the type
1552 is C{None}, we'll attempt to let C{mount} auto-detect it. This may or may
1553 not work on all systems.
1554
1555 @note: This only works on platforms that have a concept of "mounting" a
1556 filesystem through a command-line C{"mount"} command, like UNIXes. It
1557 won't work on Windows.
1558
1559 @param devicePath: Path of device to be mounted.
1560 @param mountPoint: Path that device should be mounted at.
1561 @param fsType: Type of the filesystem assumed to be available via the device.
1562
1563 @raise IOError: If the device cannot be mounted.
1564 """
1565 if fsType is None:
1566 args = [ devicePath, mountPoint ]
1567 else:
1568 args = [ "-t", fsType, devicePath, mountPoint ]
1569 command = resolveCommand(MOUNT_COMMAND)
1570 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True)[0]
1571 if result != 0:
1572 raise IOError("Error [%d] mounting [%s] at [%s] as [%s]." % (result, devicePath, mountPoint, fsType))
1573
1574
1575
1576
1577
1578
1579 -def unmount(mountPoint, removeAfter=False, attempts=1, waitSeconds=0):
1580 """
1581 Unmounts whatever device is mounted at the indicated mount point.
1582
1583 Sometimes, it might not be possible to unmount the mount point immediately,
1584 if there are still files open there. Use the C{attempts} and C{waitSeconds}
1585 arguments to indicate how many unmount attempts to make and how many seconds
1586 to wait between attempts. If you pass in zero attempts, no attempts will be
1587 made (duh).
1588
1589 If the indicated mount point is not really a mount point per
1590 C{os.path.ismount()}, then it will be ignored. This seems to be a safer
1591 check then looking through C{/etc/mtab}, since C{ismount()} is already in
1592 the Python standard library and is documented as working on all POSIX
1593 systems.
1594
1595 If C{removeAfter} is C{True}, then the mount point will be removed using
1596 C{os.rmdir()} after the unmount action succeeds. If for some reason the
1597 mount point is not a directory, then it will not be removed.
1598
1599 @note: This only works on platforms that have a concept of "mounting" a
1600 filesystem through a command-line C{"mount"} command, like UNIXes. It
1601 won't work on Windows.
1602
1603 @param mountPoint: Mount point to be unmounted.
1604 @param removeAfter: Remove the mount point after unmounting it.
1605 @param attempts: Number of times to attempt the unmount.
1606 @param waitSeconds: Number of seconds to wait between repeated attempts.
1607
1608 @raise IOError: If the mount point is still mounted after attempts are exhausted.
1609 """
1610 if os.path.ismount(mountPoint):
1611 for attempt in range(0, attempts):
1612 logger.debug("Making attempt %d to unmount [%s]." % (attempt, mountPoint))
1613 command = resolveCommand(UMOUNT_COMMAND)
1614 result = executeCommand(command, [ mountPoint, ], returnOutput=False, ignoreStderr=True)[0]
1615 if result != 0:
1616 logger.error("Error [%d] unmounting [%s] on attempt %d." % (result, mountPoint, attempt))
1617 elif os.path.ismount(mountPoint):
1618 logger.error("After attempt %d, [%s] is still mounted." % (attempt, mountPoint))
1619 else:
1620 logger.debug("Successfully unmounted [%s] on attempt %d." % (mountPoint, attempt))
1621 break
1622 if attempt+1 < attempts:
1623 if waitSeconds > 0:
1624 logger.info("Sleeping %d second(s) before next unmount attempt." % waitSeconds)
1625 time.sleep(waitSeconds)
1626 else:
1627 if os.path.ismount(mountPoint):
1628 raise IOError("Unable to unmount [%s] after %d attempts." % (mountPoint, attempts))
1629 logger.info("Mount point [%s] seems to have finally gone away." % mountPoint)
1630 if os.path.isdir(mountPoint) and removeAfter:
1631 logger.debug("Removing mount point [%s]." % mountPoint)
1632 os.rmdir(mountPoint)
1633
1640 """
1641 Indicates whether a specific filesystem device is currently mounted.
1642
1643 We determine whether the device is mounted by looking through the system's
1644 C{mtab} file. This file shows every currently-mounted filesystem, ordered
1645 by device. We only do the check if the C{mtab} file exists and is readable.
1646 Otherwise, we assume that the device is not mounted.
1647
1648 @note: This only works on platforms that have a concept of an mtab file
1649 to show mounted volumes, like UNIXes. It won't work on Windows.
1650
1651 @param devicePath: Path of device to be checked
1652
1653 @return: True if device is mounted, false otherwise.
1654 """
1655 if os.path.exists(MTAB_FILE) and os.access(MTAB_FILE, os.R_OK):
1656 realPath = os.path.realpath(devicePath)
1657 lines = open(MTAB_FILE).readlines()
1658 for line in lines:
1659 (mountDevice, mountPoint, remainder) = line.split(None, 2)
1660 if mountDevice in [ devicePath, realPath, ]:
1661 logger.debug("Device [%s] is mounted at [%s]." % (devicePath, mountPoint))
1662 return True
1663 return False
1664
1671
1672 r"""
1673 Safely encodes a filesystem path.
1674
1675 Many Python filesystem functions, such as C{os.listdir}, behave differently
1676 if they are passed unicode arguments versus simple string arguments. For
1677 instance, C{os.listdir} generally returns unicode path names if it is passed
1678 a unicode argument, and string pathnames if it is passed a string argument.
1679
1680 However, this behavior often isn't as consistent as we might like. As an example,
1681 C{os.listdir} "gives up" if it finds a filename that it can't properly encode
1682 given the current locale settings. This means that the returned list is
1683 a mixed set of unicode and simple string paths. This has consequences later,
1684 because other filesystem functions like C{os.path.join} will blow up if they
1685 are given one string path and one unicode path.
1686
1687 On comp.lang.python, Martin v. Löwis explained the C{os.listdir} behavior
1688 like this::
1689
1690 The operating system (POSIX) does not have the inherent notion that file
1691 names are character strings. Instead, in POSIX, file names are primarily
1692 byte strings. There are some bytes which are interpreted as characters
1693 (e.g. '\x2e', which is '.', or '\x2f', which is '/'), but apart from
1694 that, most OS layers think these are just bytes.
1695
1696 Now, most *people* think that file names are character strings. To
1697 interpret a file name as a character string, you need to know what the
1698 encoding is to interpret the file names (which are byte strings) as
1699 character strings.
1700
1701 There is, unfortunately, no operating system API to carry the notion of a
1702 file system encoding. By convention, the locale settings should be used
1703 to establish this encoding, in particular the LC_CTYPE facet of the
1704 locale. This is defined in the environment variables LC_CTYPE, LC_ALL,
1705 and LANG (searched in this order).
1706
1707 If LANG is not set, the "C" locale is assumed, which uses ASCII as its
1708 file system encoding. In this locale, '\xe2\x99\xaa\xe2\x99\xac' is not a
1709 valid file name (at least it cannot be interpreted as characters, and
1710 hence not be converted to Unicode).
1711
1712 Now, your Python script has requested that all file names *should* be
1713 returned as character (ie. Unicode) strings, but Python cannot comply,
1714 since there is no way to find out what this byte string means, in terms
1715 of characters.
1716
1717 So we have three options:
1718
1719 1. Skip this string, only return the ones that can be converted to Unicode.
1720 Give the user the impression the file does not exist.
1721 2. Return the string as a byte string
1722 3. Refuse to listdir altogether, raising an exception (i.e. return nothing)
1723
1724 Python has chosen alternative 2, allowing the application to implement 1
1725 or 3 on top of that if it wants to (or come up with other strategies,
1726 such as user feedback).
1727
1728 As a solution, he suggests that rather than passing unicode paths into the
1729 filesystem functions, that I should sensibly encode the path first. That is
1730 what this function accomplishes. Any function which takes a filesystem path
1731 as an argument should encode it first, before using it for any other purpose.
1732
1733 I confess I still don't completely understand how this works. On a system
1734 with filesystem encoding "ISO-8859-1", a path C{u"\xe2\x99\xaa\xe2\x99\xac"}
1735 is converted into the string C{"\xe2\x99\xaa\xe2\x99\xac"}. However, on a
1736 system with a "utf-8" encoding, the result is a completely different string:
1737 C{"\xc3\xa2\xc2\x99\xc2\xaa\xc3\xa2\xc2\x99\xc2\xac"}. A quick test where I
1738 write to the first filename and open the second proves that the two strings
1739 represent the same file on disk, which is all I really care about.
1740
1741 @note: As a special case, if C{path} is C{None}, then this function will
1742 return C{None}.
1743
1744 @note: To provide several examples of encoding values, my Debian sarge box
1745 with an ext3 filesystem has Python filesystem encoding C{ISO-8859-1}. User
1746 Anarcat's Debian box with a xfs filesystem has filesystem encoding
1747 C{ANSI_X3.4-1968}. Both my iBook G4 running Mac OS X 10.4 and user Dag
1748 Rende's SuSE 9.3 box both have filesystem encoding C{UTF-8}.
1749
1750 @note: Just because a filesystem has C{UTF-8} encoding doesn't mean that it
1751 will be able to handle all extended-character filenames. For instance,
1752 certain extended-character (but not UTF-8) filenames -- like the ones in the
1753 regression test tar file C{test/data/tree13.tar.gz} -- are not valid under
1754 Mac OS X, and it's not even possible to extract them from the tarfile on
1755 that platform.
1756
1757 @param path: Path to encode
1758
1759 @return: Path, as a string, encoded appropriately
1760 @raise ValueError: If the path cannot be encoded properly.
1761 """
1762 if path is None:
1763 return path
1764 try:
1765 if isinstance(path, unicode):
1766 encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
1767 path = path.encode(encoding)
1768 return path
1769 except UnicodeError:
1770 raise ValueError("Path could not be safely encoded as %s." % encoding)
1771
1778 """
1779 Attempts to portably return the null device on this system.
1780
1781 The null device is something like C{/dev/null} on a UNIX system. The name
1782 varies on other platforms.
1783 """
1784 return os.devnull
1785
1792 """
1793 Converts English day name to numeric day of week as from C{time.localtime}.
1794
1795 For instance, the day C{monday} would be converted to the number C{0}.
1796
1797 @param dayName: Day of week to convert
1798 @type dayName: string, i.e. C{"monday"}, C{"tuesday"}, etc.
1799
1800 @returns: Integer, where Monday is 0 and Sunday is 6; or -1 if no conversion is possible.
1801 """
1802 if dayName.lower() == "monday":
1803 return 0
1804 elif dayName.lower() == "tuesday":
1805 return 1
1806 elif dayName.lower() == "wednesday":
1807 return 2
1808 elif dayName.lower() == "thursday":
1809 return 3
1810 elif dayName.lower() == "friday":
1811 return 4
1812 elif dayName.lower() == "saturday":
1813 return 5
1814 elif dayName.lower() == "sunday":
1815 return 6
1816 else:
1817 return -1
1818
1825 """
1826 Indicates whether "today" is the backup starting day per configuration.
1827
1828 If the current day's English name matches the indicated starting day, then
1829 today is a starting day.
1830
1831 @param startingDay: Configured starting day.
1832 @type startingDay: string, i.e. C{"monday"}, C{"tuesday"}, etc.
1833
1834 @return: Boolean indicating whether today is the starting day.
1835 """
1836 value = time.localtime().tm_wday == deriveDayOfWeek(startingDay)
1837 if value:
1838 logger.debug("Today is the start of the week.")
1839 else:
1840 logger.debug("Today is NOT the start of the week.")
1841 return value
1842
1849 """
1850 Returns a "normalized" path based on a path name.
1851
1852 A normalized path is a representation of a path that is also a valid file
1853 name. To make a valid file name out of a complete path, we have to convert
1854 or remove some characters that are significant to the filesystem -- in
1855 particular, the path separator and any leading C{'.'} character (which would
1856 cause the file to be hidden in a file listing).
1857
1858 Note that this is a one-way transformation -- you can't safely derive the
1859 original path from the normalized path.
1860
1861 To normalize a path, we begin by looking at the first character. If the
1862 first character is C{'/'} or C{'\\'}, it gets removed. If the first
1863 character is C{'.'}, it gets converted to C{'_'}. Then, we look through the
1864 rest of the path and convert all remaining C{'/'} or C{'\\'} characters
1865 C{'-'}, and all remaining whitespace characters to C{'_'}.
1866
1867 As a special case, a path consisting only of a single C{'/'} or C{'\\'}
1868 character will be converted to C{'-'}.
1869
1870 @param path: Path to normalize
1871
1872 @return: Normalized path as described above.
1873
1874 @raise ValueError: If the path is None
1875 """
1876 if path is None:
1877 raise ValueError("Cannot normalize path None.")
1878 elif len(path) == 0:
1879 return path
1880 elif path == "/" or path == "\\":
1881 return "-"
1882 else:
1883 normalized = path
1884 normalized = re.sub(r"^\/", "", normalized)
1885 normalized = re.sub(r"^\\", "", normalized)
1886 normalized = re.sub(r"^\.", "_", normalized)
1887 normalized = re.sub(r"\/", "-", normalized)
1888 normalized = re.sub(r"\\", "-", normalized)
1889 normalized = re.sub(r"\s", "_", normalized)
1890 return normalized
1891
1898 """
1899 Sanitizes the operating system environment.
1900
1901 The operating system environment is contained in C{os.environ}. This method
1902 sanitizes the contents of that dictionary.
1903
1904 Currently, all it does is reset the locale (removing C{$LC_*}) and set the
1905 default language (C{$LANG}) to L{DEFAULT_LANGUAGE}. This way, we can count
1906 on consistent localization regardless of what the end-user has configured.
1907 This is important for code that needs to parse program output.
1908
1909 The C{os.environ} dictionary is modifed in-place. If C{$LANG} is already
1910 set to the proper value, it is not re-set, so we can avoid the memory leaks
1911 that are documented to occur on BSD-based systems.
1912
1913 @return: Copy of the sanitized environment.
1914 """
1915 for var in LOCALE_VARS:
1916 if os.environ.has_key(var):
1917 del os.environ[var]
1918 if os.environ.has_key(LANG_VAR):
1919 if os.environ[LANG_VAR] != DEFAULT_LANGUAGE:
1920 os.environ[LANG_VAR] = DEFAULT_LANGUAGE
1921 return os.environ.copy()
1922
1929 """
1930 Deference a soft link, optionally normalizing it to an absolute path.
1931 @param path: Path of link to dereference
1932 @param absolute: Whether to normalize the result to an absolute path
1933 @return: Dereferenced path, or original path if original is not a link.
1934 """
1935 if os.path.islink(path):
1936 result = os.readlink(path)
1937 if absolute and not os.path.isabs(result):
1938 result = os.path.abspath(os.path.join(os.path.dirname(path), result))
1939 return result
1940 return path
1941
1942
1943
1944
1945
1946
1947 -def checkUnique(prefix, values):
1948 """
1949 Checks that all values are unique.
1950
1951 The values list is checked for duplicate values. If there are
1952 duplicates, an exception is thrown. All duplicate values are listed in
1953 the exception.
1954
1955 @param prefix: Prefix to use in the thrown exception
1956 @param values: List of values to check
1957
1958 @raise ValueError: If there are duplicates in the list
1959 """
1960 values.sort()
1961 duplicates = []
1962 for i in range(1, len(values)):
1963 if values[i-1] == values[i]:
1964 duplicates.append(values[i])
1965 if duplicates:
1966 raise ValueError("%s %s" % (prefix, duplicates))
1967
1974 """
1975 Parses a list of values out of a comma-separated string.
1976
1977 The items in the list are split by comma, and then have whitespace
1978 stripped. As a special case, if C{commaString} is C{None}, then C{None}
1979 will be returned.
1980
1981 @param commaString: List of values in comma-separated string format.
1982 @return: Values from commaString split into a list, or C{None}.
1983 """
1984 if commaString is None:
1985 return None
1986 else:
1987 pass1 = commaString.split(",")
1988 pass2 = []
1989 for item in pass1:
1990 item = item.strip()
1991 if len(item) > 0:
1992 pass2.append(item)
1993 return pass2
1994