| 1 | """Conversion pipeline templates.
|
|---|
| 2 |
|
|---|
| 3 | The problem:
|
|---|
| 4 | ------------
|
|---|
| 5 |
|
|---|
| 6 | Suppose you have some data that you want to convert to another format,
|
|---|
| 7 | such as from GIF image format to PPM image format. Maybe the
|
|---|
| 8 | conversion involves several steps (e.g. piping it through compress or
|
|---|
| 9 | uuencode). Some of the conversion steps may require that their input
|
|---|
| 10 | is a disk file, others may be able to read standard input; similar for
|
|---|
| 11 | their output. The input to the entire conversion may also be read
|
|---|
| 12 | from a disk file or from an open file, and similar for its output.
|
|---|
| 13 |
|
|---|
| 14 | The module lets you construct a pipeline template by sticking one or
|
|---|
| 15 | more conversion steps together. It will take care of creating and
|
|---|
| 16 | removing temporary files if they are necessary to hold intermediate
|
|---|
| 17 | data. You can then use the template to do conversions from many
|
|---|
| 18 | different sources to many different destinations. The temporary
|
|---|
| 19 | file names used are different each time the template is used.
|
|---|
| 20 |
|
|---|
| 21 | The templates are objects so you can create templates for many
|
|---|
| 22 | different conversion steps and store them in a dictionary, for
|
|---|
| 23 | instance.
|
|---|
| 24 |
|
|---|
| 25 |
|
|---|
| 26 | Directions:
|
|---|
| 27 | -----------
|
|---|
| 28 |
|
|---|
| 29 | To create a template:
|
|---|
| 30 | t = Template()
|
|---|
| 31 |
|
|---|
| 32 | To add a conversion step to a template:
|
|---|
| 33 | t.append(command, kind)
|
|---|
| 34 | where kind is a string of two characters: the first is '-' if the
|
|---|
| 35 | command reads its standard input or 'f' if it requires a file; the
|
|---|
| 36 | second likewise for the output. The command must be valid /bin/sh
|
|---|
| 37 | syntax. If input or output files are required, they are passed as
|
|---|
| 38 | $IN and $OUT; otherwise, it must be possible to use the command in
|
|---|
| 39 | a pipeline.
|
|---|
| 40 |
|
|---|
| 41 | To add a conversion step at the beginning:
|
|---|
| 42 | t.prepend(command, kind)
|
|---|
| 43 |
|
|---|
| 44 | To convert a file to another file using a template:
|
|---|
| 45 | sts = t.copy(infile, outfile)
|
|---|
| 46 | If infile or outfile are the empty string, standard input is read or
|
|---|
| 47 | standard output is written, respectively. The return value is the
|
|---|
| 48 | exit status of the conversion pipeline.
|
|---|
| 49 |
|
|---|
| 50 | To open a file for reading or writing through a conversion pipeline:
|
|---|
| 51 | fp = t.open(file, mode)
|
|---|
| 52 | where mode is 'r' to read the file, or 'w' to write it -- just like
|
|---|
| 53 | for the built-in function open() or for os.popen().
|
|---|
| 54 |
|
|---|
| 55 | To create a new template object initialized to a given one:
|
|---|
| 56 | t2 = t.clone()
|
|---|
| 57 |
|
|---|
| 58 | For an example, see the function test() at the end of the file.
|
|---|
| 59 | """ # '
|
|---|
| 60 |
|
|---|
| 61 |
|
|---|
| 62 | import re
|
|---|
| 63 |
|
|---|
| 64 | import os
|
|---|
| 65 | import tempfile
|
|---|
| 66 | import string
|
|---|
| 67 |
|
|---|
| 68 | __all__ = ["Template"]
|
|---|
| 69 |
|
|---|
| 70 | # Conversion step kinds
|
|---|
| 71 |
|
|---|
| 72 | FILEIN_FILEOUT = 'ff' # Must read & write real files
|
|---|
| 73 | STDIN_FILEOUT = '-f' # Must write a real file
|
|---|
| 74 | FILEIN_STDOUT = 'f-' # Must read a real file
|
|---|
| 75 | STDIN_STDOUT = '--' # Normal pipeline element
|
|---|
| 76 | SOURCE = '.-' # Must be first, writes stdout
|
|---|
| 77 | SINK = '-.' # Must be last, reads stdin
|
|---|
| 78 |
|
|---|
| 79 | stepkinds = [FILEIN_FILEOUT, STDIN_FILEOUT, FILEIN_STDOUT, STDIN_STDOUT, \
|
|---|
| 80 | SOURCE, SINK]
|
|---|
| 81 |
|
|---|
| 82 |
|
|---|
| 83 | class Template:
|
|---|
| 84 | """Class representing a pipeline template."""
|
|---|
| 85 |
|
|---|
| 86 | def __init__(self):
|
|---|
| 87 | """Template() returns a fresh pipeline template."""
|
|---|
| 88 | self.debugging = 0
|
|---|
| 89 | self.reset()
|
|---|
| 90 |
|
|---|
| 91 | def __repr__(self):
|
|---|
| 92 | """t.__repr__() implements repr(t)."""
|
|---|
| 93 | return '<Template instance, steps=%r>' % (self.steps,)
|
|---|
| 94 |
|
|---|
| 95 | def reset(self):
|
|---|
| 96 | """t.reset() restores a pipeline template to its initial state."""
|
|---|
| 97 | self.steps = []
|
|---|
| 98 |
|
|---|
| 99 | def clone(self):
|
|---|
| 100 | """t.clone() returns a new pipeline template with identical
|
|---|
| 101 | initial state as the current one."""
|
|---|
| 102 | t = Template()
|
|---|
| 103 | t.steps = self.steps[:]
|
|---|
| 104 | t.debugging = self.debugging
|
|---|
| 105 | return t
|
|---|
| 106 |
|
|---|
| 107 | def debug(self, flag):
|
|---|
| 108 | """t.debug(flag) turns debugging on or off."""
|
|---|
| 109 | self.debugging = flag
|
|---|
| 110 |
|
|---|
| 111 | def append(self, cmd, kind):
|
|---|
| 112 | """t.append(cmd, kind) adds a new step at the end."""
|
|---|
| 113 | if type(cmd) is not type(''):
|
|---|
| 114 | raise TypeError, \
|
|---|
| 115 | 'Template.append: cmd must be a string'
|
|---|
| 116 | if kind not in stepkinds:
|
|---|
| 117 | raise ValueError, \
|
|---|
| 118 | 'Template.append: bad kind %r' % (kind,)
|
|---|
| 119 | if kind == SOURCE:
|
|---|
| 120 | raise ValueError, \
|
|---|
| 121 | 'Template.append: SOURCE can only be prepended'
|
|---|
| 122 | if self.steps and self.steps[-1][1] == SINK:
|
|---|
| 123 | raise ValueError, \
|
|---|
| 124 | 'Template.append: already ends with SINK'
|
|---|
| 125 | if kind[0] == 'f' and not re.search(r'\$IN\b', cmd):
|
|---|
| 126 | raise ValueError, \
|
|---|
| 127 | 'Template.append: missing $IN in cmd'
|
|---|
| 128 | if kind[1] == 'f' and not re.search(r'\$OUT\b', cmd):
|
|---|
| 129 | raise ValueError, \
|
|---|
| 130 | 'Template.append: missing $OUT in cmd'
|
|---|
| 131 | self.steps.append((cmd, kind))
|
|---|
| 132 |
|
|---|
| 133 | def prepend(self, cmd, kind):
|
|---|
| 134 | """t.prepend(cmd, kind) adds a new step at the front."""
|
|---|
| 135 | if type(cmd) is not type(''):
|
|---|
| 136 | raise TypeError, \
|
|---|
| 137 | 'Template.prepend: cmd must be a string'
|
|---|
| 138 | if kind not in stepkinds:
|
|---|
| 139 | raise ValueError, \
|
|---|
| 140 | 'Template.prepend: bad kind %r' % (kind,)
|
|---|
| 141 | if kind == SINK:
|
|---|
| 142 | raise ValueError, \
|
|---|
| 143 | 'Template.prepend: SINK can only be appended'
|
|---|
| 144 | if self.steps and self.steps[0][1] == SOURCE:
|
|---|
| 145 | raise ValueError, \
|
|---|
| 146 | 'Template.prepend: already begins with SOURCE'
|
|---|
| 147 | if kind[0] == 'f' and not re.search(r'\$IN\b', cmd):
|
|---|
| 148 | raise ValueError, \
|
|---|
| 149 | 'Template.prepend: missing $IN in cmd'
|
|---|
| 150 | if kind[1] == 'f' and not re.search(r'\$OUT\b', cmd):
|
|---|
| 151 | raise ValueError, \
|
|---|
| 152 | 'Template.prepend: missing $OUT in cmd'
|
|---|
| 153 | self.steps.insert(0, (cmd, kind))
|
|---|
| 154 |
|
|---|
| 155 | def open(self, file, rw):
|
|---|
| 156 | """t.open(file, rw) returns a pipe or file object open for
|
|---|
| 157 | reading or writing; the file is the other end of the pipeline."""
|
|---|
| 158 | if rw == 'r':
|
|---|
| 159 | return self.open_r(file)
|
|---|
| 160 | if rw == 'w':
|
|---|
| 161 | return self.open_w(file)
|
|---|
| 162 | raise ValueError, \
|
|---|
| 163 | 'Template.open: rw must be \'r\' or \'w\', not %r' % (rw,)
|
|---|
| 164 |
|
|---|
| 165 | def open_r(self, file):
|
|---|
| 166 | """t.open_r(file) and t.open_w(file) implement
|
|---|
| 167 | t.open(file, 'r') and t.open(file, 'w') respectively."""
|
|---|
| 168 | if not self.steps:
|
|---|
| 169 | return open(file, 'r')
|
|---|
| 170 | if self.steps[-1][1] == SINK:
|
|---|
| 171 | raise ValueError, \
|
|---|
| 172 | 'Template.open_r: pipeline ends width SINK'
|
|---|
| 173 | cmd = self.makepipeline(file, '')
|
|---|
| 174 | return os.popen(cmd, 'r')
|
|---|
| 175 |
|
|---|
| 176 | def open_w(self, file):
|
|---|
| 177 | if not self.steps:
|
|---|
| 178 | return open(file, 'w')
|
|---|
| 179 | if self.steps[0][1] == SOURCE:
|
|---|
| 180 | raise ValueError, \
|
|---|
| 181 | 'Template.open_w: pipeline begins with SOURCE'
|
|---|
| 182 | cmd = self.makepipeline('', file)
|
|---|
| 183 | return os.popen(cmd, 'w')
|
|---|
| 184 |
|
|---|
| 185 | def copy(self, infile, outfile):
|
|---|
| 186 | return os.system(self.makepipeline(infile, outfile))
|
|---|
| 187 |
|
|---|
| 188 | def makepipeline(self, infile, outfile):
|
|---|
| 189 | cmd = makepipeline(infile, self.steps, outfile)
|
|---|
| 190 | if self.debugging:
|
|---|
| 191 | print cmd
|
|---|
| 192 | cmd = 'set -x; ' + cmd
|
|---|
| 193 | return cmd
|
|---|
| 194 |
|
|---|
| 195 |
|
|---|
| 196 | def makepipeline(infile, steps, outfile):
|
|---|
| 197 | # Build a list with for each command:
|
|---|
| 198 | # [input filename or '', command string, kind, output filename or '']
|
|---|
| 199 |
|
|---|
| 200 | list = []
|
|---|
| 201 | for cmd, kind in steps:
|
|---|
| 202 | list.append(['', cmd, kind, ''])
|
|---|
| 203 | #
|
|---|
| 204 | # Make sure there is at least one step
|
|---|
| 205 | #
|
|---|
| 206 | if not list:
|
|---|
| 207 | list.append(['', 'cat', '--', ''])
|
|---|
| 208 | #
|
|---|
| 209 | # Take care of the input and output ends
|
|---|
| 210 | #
|
|---|
| 211 | [cmd, kind] = list[0][1:3]
|
|---|
| 212 | if kind[0] == 'f' and not infile:
|
|---|
| 213 | list.insert(0, ['', 'cat', '--', ''])
|
|---|
| 214 | list[0][0] = infile
|
|---|
| 215 | #
|
|---|
| 216 | [cmd, kind] = list[-1][1:3]
|
|---|
| 217 | if kind[1] == 'f' and not outfile:
|
|---|
| 218 | list.append(['', 'cat', '--', ''])
|
|---|
| 219 | list[-1][-1] = outfile
|
|---|
| 220 | #
|
|---|
| 221 | # Invent temporary files to connect stages that need files
|
|---|
| 222 | #
|
|---|
| 223 | garbage = []
|
|---|
| 224 | for i in range(1, len(list)):
|
|---|
| 225 | lkind = list[i-1][2]
|
|---|
| 226 | rkind = list[i][2]
|
|---|
| 227 | if lkind[1] == 'f' or rkind[0] == 'f':
|
|---|
| 228 | (fd, temp) = tempfile.mkstemp()
|
|---|
| 229 | os.close(fd)
|
|---|
| 230 | garbage.append(temp)
|
|---|
| 231 | list[i-1][-1] = list[i][0] = temp
|
|---|
| 232 | #
|
|---|
| 233 | for item in list:
|
|---|
| 234 | [inf, cmd, kind, outf] = item
|
|---|
| 235 | if kind[1] == 'f':
|
|---|
| 236 | cmd = 'OUT=' + quote(outf) + '; ' + cmd
|
|---|
| 237 | if kind[0] == 'f':
|
|---|
| 238 | cmd = 'IN=' + quote(inf) + '; ' + cmd
|
|---|
| 239 | if kind[0] == '-' and inf:
|
|---|
| 240 | cmd = cmd + ' <' + quote(inf)
|
|---|
| 241 | if kind[1] == '-' and outf:
|
|---|
| 242 | cmd = cmd + ' >' + quote(outf)
|
|---|
| 243 | item[1] = cmd
|
|---|
| 244 | #
|
|---|
| 245 | cmdlist = list[0][1]
|
|---|
| 246 | for item in list[1:]:
|
|---|
| 247 | [cmd, kind] = item[1:3]
|
|---|
| 248 | if item[0] == '':
|
|---|
| 249 | if 'f' in kind:
|
|---|
| 250 | cmd = '{ ' + cmd + '; }'
|
|---|
| 251 | cmdlist = cmdlist + ' |\n' + cmd
|
|---|
| 252 | else:
|
|---|
| 253 | cmdlist = cmdlist + '\n' + cmd
|
|---|
| 254 | #
|
|---|
| 255 | if garbage:
|
|---|
| 256 | rmcmd = 'rm -f'
|
|---|
| 257 | for file in garbage:
|
|---|
| 258 | rmcmd = rmcmd + ' ' + quote(file)
|
|---|
| 259 | trapcmd = 'trap ' + quote(rmcmd + '; exit') + ' 1 2 3 13 14 15'
|
|---|
| 260 | cmdlist = trapcmd + '\n' + cmdlist + '\n' + rmcmd
|
|---|
| 261 | #
|
|---|
| 262 | return cmdlist
|
|---|
| 263 |
|
|---|
| 264 |
|
|---|
| 265 | # Reliably quote a string as a single argument for /bin/sh
|
|---|
| 266 |
|
|---|
| 267 | _safechars = string.ascii_letters + string.digits + '!@%_-+=:,./' # Safe unquoted
|
|---|
| 268 | _funnychars = '"`$\\' # Unsafe inside "double quotes"
|
|---|
| 269 |
|
|---|
| 270 | def quote(file):
|
|---|
| 271 | for c in file:
|
|---|
| 272 | if c not in _safechars:
|
|---|
| 273 | break
|
|---|
| 274 | else:
|
|---|
| 275 | return file
|
|---|
| 276 | if '\'' not in file:
|
|---|
| 277 | return '\'' + file + '\''
|
|---|
| 278 | res = ''
|
|---|
| 279 | for c in file:
|
|---|
| 280 | if c in _funnychars:
|
|---|
| 281 | c = '\\' + c
|
|---|
| 282 | res = res + c
|
|---|
| 283 | return '"' + res + '"'
|
|---|
| 284 |
|
|---|
| 285 |
|
|---|
| 286 | # Small test program and example
|
|---|
| 287 |
|
|---|
| 288 | def test():
|
|---|
| 289 | print 'Testing...'
|
|---|
| 290 | t = Template()
|
|---|
| 291 | t.append('togif $IN $OUT', 'ff')
|
|---|
| 292 | t.append('giftoppm', '--')
|
|---|
| 293 | t.append('ppmtogif >$OUT', '-f')
|
|---|
| 294 | t.append('fromgif $IN $OUT', 'ff')
|
|---|
| 295 | t.debug(1)
|
|---|
| 296 | FILE = '/usr/local/images/rgb/rogues/guido.rgb'
|
|---|
| 297 | t.copy(FILE, '@temp')
|
|---|
| 298 | print 'Done.'
|
|---|
