Package logilab :: Package common :: Module fileutils
[frames] | no frames]

Source Code for Module logilab.common.fileutils

  1  # copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved. 
  2  # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr 
  3  # 
  4  # This file is part of logilab-common. 
  5  # 
  6  # logilab-common is free software: you can redistribute it and/or modify it under 
  7  # the terms of the GNU Lesser General Public License as published by the Free 
  8  # Software Foundation, either version 2.1 of the License, or (at your option) any 
  9  # later version. 
 10  # 
 11  # logilab-common is distributed in the hope that it will be useful, but WITHOUT 
 12  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
 13  # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more 
 14  # details. 
 15  # 
 16  # You should have received a copy of the GNU Lesser General Public License along 
 17  # with logilab-common.  If not, see <http://www.gnu.org/licenses/>. 
 18  """File and file-path manipulation utilities. 
 19   
 20  :group path manipulation: first_level_directory, relative_path, is_binary,\ 
 21  get_by_ext, remove_dead_links 
 22  :group file manipulation: norm_read, norm_open, lines, stream_lines, lines,\ 
 23  write_open_mode, ensure_fs_mode, export 
 24  :sort: path manipulation, file manipulation 
 25  """ 
 26  __docformat__ = "restructuredtext en" 
 27   
 28  import sys 
 29  import shutil 
 30  import mimetypes 
 31  from os.path import isabs, isdir, islink, split, exists, normpath, join 
 32  from os.path import abspath 
 33  from os import sep, mkdir, remove, listdir, stat, chmod, walk 
 34  from stat import ST_MODE, S_IWRITE 
 35  from cStringIO import StringIO 
 36   
 37  from logilab.common import STD_BLACKLIST as BASE_BLACKLIST, IGNORED_EXTENSIONS 
 38  from logilab.common.shellutils import find 
 39  from logilab.common.deprecation import deprecated 
 40  from logilab.common.compat import FileIO, any 
 41   
42 -def first_level_directory(path):
43 """Return the first level directory of a path. 44 45 >>> first_level_directory('home/syt/work') 46 'home' 47 >>> first_level_directory('/home/syt/work') 48 '/' 49 >>> first_level_directory('work') 50 'work' 51 >>> 52 53 :type path: str 54 :param path: the path for which we want the first level directory 55 56 :rtype: str 57 :return: the first level directory appearing in `path` 58 """ 59 head, tail = split(path) 60 while head and tail: 61 head, tail = split(head) 62 if tail: 63 return tail 64 # path was absolute, head is the fs root 65 return head
66
67 -def abspath_listdir(path):
68 """Lists path's content using absolute paths. 69 70 >>> os.listdir('/home') 71 ['adim', 'alf', 'arthur', 'auc'] 72 >>> abspath_listdir('/home') 73 ['/home/adim', '/home/alf', '/home/arthur', '/home/auc'] 74 """ 75 path = abspath(path) 76 return [join(path, filename) for filename in listdir(path)]
77 78
79 -def is_binary(filename):
80 """Return true if filename may be a binary file, according to it's 81 extension. 82 83 :type filename: str 84 :param filename: the name of the file 85 86 :rtype: bool 87 :return: 88 true if the file is a binary file (actually if it's mime type 89 isn't beginning by text/) 90 """ 91 try: 92 return not mimetypes.guess_type(filename)[0].startswith('text') 93 except AttributeError: 94 return 1
95 96
97 -def write_open_mode(filename):
98 """Return the write mode that should used to open file. 99 100 :type filename: str 101 :param filename: the name of the file 102 103 :rtype: str 104 :return: the mode that should be use to open the file ('w' or 'wb') 105 """ 106 if is_binary(filename): 107 return 'wb' 108 return 'w'
109 110
111 -def ensure_fs_mode(filepath, desired_mode=S_IWRITE):
112 """Check that the given file has the given mode(s) set, else try to 113 set it. 114 115 :type filepath: str 116 :param filepath: path of the file 117 118 :type desired_mode: int 119 :param desired_mode: 120 ORed flags describing the desired mode. Use constants from the 121 `stat` module for file permission's modes 122 """ 123 mode = stat(filepath)[ST_MODE] 124 if not mode & desired_mode: 125 chmod(filepath, mode | desired_mode)
126 127 128 # XXX (syt) unused? kill?
129 -class ProtectedFile(FileIO):
130 """A special file-object class that automatically does a 'chmod +w' when 131 needed. 132 133 XXX: for now, the way it is done allows 'normal file-objects' to be 134 created during the ProtectedFile object lifetime. 135 One way to circumvent this would be to chmod / unchmod on each 136 write operation. 137 138 One other way would be to : 139 140 - catch the IOError in the __init__ 141 142 - if IOError, then create a StringIO object 143 144 - each write operation writes in this StringIO object 145 146 - on close()/del(), write/append the StringIO content to the file and 147 do the chmod only once 148 """
149 - def __init__(self, filepath, mode):
150 self.original_mode = stat(filepath)[ST_MODE] 151 self.mode_changed = False 152 if mode in ('w', 'a', 'wb', 'ab'): 153 if not self.original_mode & S_IWRITE: 154 chmod(filepath, self.original_mode | S_IWRITE) 155 self.mode_changed = True 156 FileIO.__init__(self, filepath, mode)
157
158 - def _restore_mode(self):
159 """restores the original mode if needed""" 160 if self.mode_changed: 161 chmod(self.name, self.original_mode) 162 # Don't re-chmod in case of several restore 163 self.mode_changed = False
164
165 - def close(self):
166 """restore mode before closing""" 167 self._restore_mode() 168 FileIO.close(self)
169
170 - def __del__(self):
171 if not self.closed: 172 self.close()
173 174
175 -class UnresolvableError(Exception):
176 """Exception raised by relative path when it's unable to compute relative 177 path between two paths. 178 """
179
180 -def relative_path(from_file, to_file):
181 """Try to get a relative path from `from_file` to `to_file` 182 (path will be absolute if to_file is an absolute file). This function 183 is useful to create link in `from_file` to `to_file`. This typical use 184 case is used in this function description. 185 186 If both files are relative, they're expected to be relative to the same 187 directory. 188 189 >>> relative_path( from_file='toto/index.html', to_file='index.html') 190 '../index.html' 191 >>> relative_path( from_file='index.html', to_file='toto/index.html') 192 'toto/index.html' 193 >>> relative_path( from_file='tutu/index.html', to_file='toto/index.html') 194 '../toto/index.html' 195 >>> relative_path( from_file='toto/index.html', to_file='/index.html') 196 '/index.html' 197 >>> relative_path( from_file='/toto/index.html', to_file='/index.html') 198 '../index.html' 199 >>> relative_path( from_file='/toto/index.html', to_file='/toto/summary.html') 200 'summary.html' 201 >>> relative_path( from_file='index.html', to_file='index.html') 202 '' 203 >>> relative_path( from_file='/index.html', to_file='toto/index.html') 204 Traceback (most recent call last): 205 File "<string>", line 1, in ? 206 File "<stdin>", line 37, in relative_path 207 UnresolvableError 208 >>> relative_path( from_file='/index.html', to_file='/index.html') 209 '' 210 >>> 211 212 :type from_file: str 213 :param from_file: source file (where links will be inserted) 214 215 :type to_file: str 216 :param to_file: target file (on which links point) 217 218 :raise UnresolvableError: if it has been unable to guess a correct path 219 220 :rtype: str 221 :return: the relative path of `to_file` from `from_file` 222 """ 223 from_file = normpath(from_file) 224 to_file = normpath(to_file) 225 if from_file == to_file: 226 return '' 227 if isabs(to_file): 228 if not isabs(from_file): 229 return to_file 230 elif isabs(from_file): 231 raise UnresolvableError() 232 from_parts = from_file.split(sep) 233 to_parts = to_file.split(sep) 234 idem = 1 235 result = [] 236 while len(from_parts) > 1: 237 dirname = from_parts.pop(0) 238 if idem and len(to_parts) > 1 and dirname == to_parts[0]: 239 to_parts.pop(0) 240 else: 241 idem = 0 242 result.append('..') 243 result += to_parts 244 return sep.join(result)
245 246
247 -def norm_read(path):
248 """Return the content of the file with normalized line feeds. 249 250 :type path: str 251 :param path: path to the file to read 252 253 :rtype: str 254 :return: the content of the file with normalized line feeds 255 """ 256 return open(path, 'U').read()
257 norm_read = deprecated("use \"open(path, 'U').read()\"")(norm_read) 258
259 -def norm_open(path):
260 """Return a stream for a file with content with normalized line feeds. 261 262 :type path: str 263 :param path: path to the file to open 264 265 :rtype: file or StringIO 266 :return: the opened file with normalized line feeds 267 """ 268 return open(path, 'U')
269 norm_open = deprecated("use \"open(path, 'U')\"")(norm_open) 270
271 -def lines(path, comments=None):
272 """Return a list of non empty lines in the file located at `path`. 273 274 :type path: str 275 :param path: path to the file 276 277 :type comments: str or None 278 :param comments: 279 optional string which can be used to comment a line in the file 280 (i.e. lines starting with this string won't be returned) 281 282 :rtype: list 283 :return: 284 a list of stripped line in the file, without empty and commented 285 lines 286 287 :warning: at some point this function will probably return an iterator 288 """ 289 stream = open(path, 'U') 290 result = stream_lines(stream, comments) 291 stream.close() 292 return result
293 294
295 -def stream_lines(stream, comments=None):
296 """Return a list of non empty lines in the given `stream`. 297 298 :type stream: object implementing 'xreadlines' or 'readlines' 299 :param stream: file like object 300 301 :type comments: str or None 302 :param comments: 303 optional string which can be used to comment a line in the file 304 (i.e. lines starting with this string won't be returned) 305 306 :rtype: list 307 :return: 308 a list of stripped line in the file, without empty and commented 309 lines 310 311 :warning: at some point this function will probably return an iterator 312 """ 313 try: 314 readlines = stream.xreadlines 315 except AttributeError: 316 readlines = stream.readlines 317 result = [] 318 for line in readlines(): 319 line = line.strip() 320 if line and (comments is None or not line.startswith(comments)): 321 result.append(line) 322 return result
323 324
325 -def export(from_dir, to_dir, 326 blacklist=BASE_BLACKLIST, ignore_ext=IGNORED_EXTENSIONS, 327 verbose=0):
328 """Make a mirror of `from_dir` in `to_dir`, omitting directories and 329 files listed in the black list or ending with one of the given 330 extensions. 331 332 :type from_dir: str 333 :param from_dir: directory to export 334 335 :type to_dir: str 336 :param to_dir: destination directory 337 338 :type blacklist: list or tuple 339 :param blacklist: 340 list of files or directories to ignore, default to the content of 341 `BASE_BLACKLIST` 342 343 :type ignore_ext: list or tuple 344 :param ignore_ext: 345 list of extensions to ignore, default to the content of 346 `IGNORED_EXTENSIONS` 347 348 :type verbose: bool 349 :param verbose: 350 flag indicating whether information about exported files should be 351 printed to stderr, default to False 352 """ 353 try: 354 mkdir(to_dir) 355 except OSError: 356 pass # FIXME we should use "exists" if the point is about existing dir 357 # else (permission problems?) shouldn't return / raise ? 358 for directory, dirnames, filenames in walk(from_dir): 359 for norecurs in blacklist: 360 try: 361 dirnames.remove(norecurs) 362 except ValueError: 363 continue 364 for dirname in dirnames: 365 src = join(directory, dirname) 366 dest = to_dir + src[len(from_dir):] 367 if isdir(src): 368 if not exists(dest): 369 mkdir(dest) 370 for filename in filenames: 371 # don't include binary files 372 # endswith does not accept tuple in 2.4 373 if any([filename.endswith(ext) for ext in ignore_ext]): 374 continue 375 src = join(directory, filename) 376 dest = to_dir + src[len(from_dir):] 377 if verbose: 378 print >> sys.stderr, src, '->', dest 379 if exists(dest): 380 remove(dest) 381 shutil.copy2(src, dest)
382 383 402