from IPython.display import HTML
HTML(open("../include/notes.css", "r").read())
Like any other program, your program needs to interact with the operating system. We've already seen one aspect of this: I/O. There are others.
argparse Module¶All programs have a user interface (UI). Those with a command line interface (CLI) need ways to specify options, input files, and the like. One way to do this, taught to novice programmers, is to prompt the user, but this gets tedious after a while and makes scripting difficult. Definitely unpythonic, if not uncouth.
You can always parse sys.argv[] yourself, but less time consuming and more consistent to use a Python module to do it:
In the beginning, there was getopt, very much like the POSIX getopt(3) (with GNU extensions, cf. getopt_longer(3)).
getopt begat optparse, which parsed "options" (but left what came after up to the user). It also did cool stuff like generate consistent help messages automatically.
Nowadays, there's argparse, which, as the name sugggests, does it all.
Here's a simple program using argparse:
%%file demos/d00_argparse_simple.py
import argparse
argumentParser = argparse.ArgumentParser()
argumentParser.parse_args()
If we run it by itself, it does nothing.
!python3 demos/d00_argparse_simple.py
but you see there's already a self-referential help message available with both a short ...
!python3 demos/d00_argparse_simple.py -h
... and a long form:
!python3 demos/d00_argparse_simple.py --help
And if we try a command line that contains an argument:
!python3 demos/d00_argparse_simple.py spam
Note that the error message went to standard error. (We can demonstrate this by using redirection above.)
Let's add an argument -- a positional one:
%%file demos/d01_argparse_only_arg.py
import argparse
argumentParser = argparse.ArgumentParser()
argumentParser.add_argument("onlyArg")
args = argumentParser.parse_args()
print("args.onlyArg:", args.onlyArg)
!python3 demos/d01_argparse_only_arg.py foo
So the argument is now legal and the name you give it in add_argument() is used as an attribute of the return from parse_args(). The value of that attribute is the (str) argument.
Now, if we don't pass an argument, we get an error:
!python3 demos/d01_argparse_only_arg.py
The argument is automatically added to the help message as well:
!python3 demos/d01_argparse_only_arg.py --help
But we can make this help message more -- uh -- helpful by attaching text to the argument.
%%file demos/d02_argparse_only_arg_helpful.py
import argparse
argumentParser = argparse.ArgumentParser()
argumentParser.add_argument("onlyArg",
help="the only argument (add description as needed)")
args = argumentParser.parse_args()
print("args.onlyArg:", args.onlyArg)
%%sh
python3 demos/d02_argparse_only_arg_helpful.py --help
By default, the argument is a string, but we can make parse_args() convert it to another type. Let's write a program to do something useful with this. In addition to "help" keywords for each argument, we'll add an overall "description".
%%file demos/d03_argparse_quad_soln.py
import argparse
argumentParser = argparse.ArgumentParser(description=r"""
prints solutions (roots) of the quadratic equation a x**2 + b x + c = 0
""")
argumentParser.add_argument("a", help="quadratic coeffcient", type=float)
argumentParser.add_argument("b", help="linear coeffcient", type=float)
argumentParser.add_argument("c", help="constant", type=float)
args = argumentParser.parse_args()
discr = args.b**2 - 4*args.a*args.c
root1 = (-args.b + discr**0.5)/(2*args.a)
root2 = (-args.b - discr**0.5)/(2*args.a)
print("roots: {}, {}".format(root1, root2))
This works:
!python3 demos/d03_argparse_quad_soln.py 1 -7 12
If the argument can't be parsed as a float, we get an error:
!python3 demos/d03_argparse_quad_soln.py 1 -7 spam
And here's the help message:
!python3 demos/d03_argparse_quad_soln.py --help
We have one optional argument (the automatically-added "--help"). How can we add more? For starters, we indicate them with a leading "--".
%%file demos/d04_argparse_quad_soln_verbose.py
import argparse
argumentParser = argparse.ArgumentParser(description=r"""
prints solutions (roots) of the quadratic equation a x**2 + b x + c = 0
""")
argumentParser.add_argument("a",
help="quadratic coeffcient", type=float)
argumentParser.add_argument("b",
help="linear coeffcient", type=float)
argumentParser.add_argument("c",
help="constant coeffcient", type=float)
argumentParser.add_argument("--verbosity", # <- like this
help="degree of verbosity", type=int)
args = argumentParser.parse_args()
discr = args.b**2 - 4*args.a*args.c
root1 = (-args.b + discr**0.5)/(2*args.a)
root2 = (-args.b - discr**0.5)/(2*args.a)
if args.verbosity:
print("{:g}x**2{:+g}x{:+g} = (x{:+g})(x{:+g})".format(
args.a, args.b, args.c, -root1, -root2))
else:
print("roots: {}, {}".format(root1, root2))
%%sh
python3 demos/d04_argparse_quad_soln_verbose.py --verbosity 1 1 -7 12
%%sh
python3 demos/d04_argparse_quad_soln_verbose.py --help
This requires us to provide a value ("VERBOSITY") after the option flag. What if we don't want to do that?
action"¶If we want the presence of the option to act like a boolean flag, we only need to make a few small changes, including setting the "action" to "store_true" so that the result can be used as a bool.
%%file demos/d05_argparse_quad_soln_verbose_flag.py
import argparse
argumentParser = argparse.ArgumentParser(description=r"""
prints solutions (roots) of the quadratic equation a x**2 + b x + c = 0
""")
argumentParser.add_argument("a",
help="quadratic coeffcient", type=float)
argumentParser.add_argument("b",
help="linear coeffcient", type=float)
argumentParser.add_argument("c",
help="constant coeffcient", type=float)
argumentParser.add_argument("--verbose", # <- name change
action='store_true', # <- added
help="verbose output") # <- minor change
args = argumentParser.parse_args()
discr = args.b**2 - 4*args.a*args.c
root1 = (-args.b + discr**0.5)/(2*args.a)
root2 = (-args.b - discr**0.5)/(2*args.a)
if args.verbose:
print("{:g}x**2{:+g}x{:+g} = (x{:+g})(x{:+g})".format(
args.a, args.b, args.c, -root1, -root2))
else:
print("roots: {}, {}".format(root1, root2))
And so "--verbose" no longer needs an argument:
!python3 demos/d05_argparse_quad_soln_verbose_flag.py --verbose 1 -7 12
The help message reflects this:
!python3 demos/d05_argparse_quad_soln_verbose_flag.py --help
Long keyword "--"-prefixed options are recommended for scripting purposes, but short, single-character options are quicker to type and there are often conventions that make them easy to remember: "-h" is equivalent to "--help":
%%file demos/d06_argparse_quad_soln_verbose_short.py
import argparse
argumentParser = argparse.ArgumentParser(description=r"""
prints solutions (roots) of the quadratic equation a x**2 + b x + c = 0
""")
argumentParser.add_argument("a",
help="quadratic coeffcient", type=float)
argumentParser.add_argument("b",
help="linear coeffcient", type=float)
argumentParser.add_argument("c",
help="constant coeffcient", type=float)
argumentParser.add_argument("-v", "--verbose", # note new 1st argument
action='store_true',
help="verbose output")
args = argumentParser.parse_args()
discr = args.b**2 - 4*args.a*args.c
root1 = (-args.b + discr**0.5)/(2*args.a)
root2 = (-args.b - discr**0.5)/(2*args.a)
if args.verbose:
print("{:g}x**2{:+g}x{:+g} = (x{:+g})(x{:+g})".format(
args.a, args.b, args.c, -root1, -root2))
else:
print("roots: {}, {}".format(root1, root2))
The result is quicker to type:
%%sh
python3 demos/d06_argparse_quad_soln_verbose_short.py -v 1 -7 12
The "--verbose" flag continues to work, as shown in the help message:
%%sh
python3 demos/d06_argparse_quad_soln_verbose_short.py -h
How about adding a "secret" argument? One that works, but doesn't appear in the help message? Easy: Just make the "help" keyword "argparse.SUPPRESS". Let's do this for the "--verbose" argument:
%%file demos/d07_argparse_quad_soln_hidden_argument.py
import argparse
argumentParser = argparse.ArgumentParser(description=r"""
prints solutions (roots) of the quadratic equation a x**2 + b x + c = 0
""")
argumentParser.add_argument("a",
help="quadratic coeffcient", type=float)
argumentParser.add_argument("b",
help="linear coeffcient", type=float)
argumentParser.add_argument("c",
help="constant coeffcient", type=float)
argumentParser.add_argument("-v", "--verbose",
action='store_true',
help=argparse.SUPPRESS) # <- omitted from help
args = argumentParser.parse_args()
discr = args.b**2 - 4*args.a*args.c
root1 = (-args.b + discr**0.5)/(2*args.a)
root2 = (-args.b - discr**0.5)/(2*args.a)
if args.verbose:
print("{:g}x**2{:+g}x{:+g} = (x{:+g})(x{:+g})".format(
args.a, args.b, args.c, -root1, -root2))
else:
print("roots: {}, {}".format(root1, root2))
Sure enough, the "--verbose" doesn't appear in the help message:
!python3 demos/d07_argparse_quad_soln_hidden_argument.py -h
But it's still parsed:
!python3 demos/d07_argparse_quad_soln_hidden_argument.py -v 1 -7 12
Sometimes options can be mutually exclusive. Suppose there was a "--quiet" (or "-q" for short) option that made no sense to select at the same time as a "--verbose" flag. Here's how we do that:
%%file demos/d08_argparse_quad_soln_verbose_quiet.py
import argparse
argumentParser = argparse.ArgumentParser(description=r"""
prints solutions (roots) of the quadratic equation a x**2 + b x + c = 0
""")
argumentParser.add_argument("a",
help="quadratic coeffcient", type=float)
argumentParser.add_argument("b",
help="linear coeffcient", type=float)
argumentParser.add_argument("c",
help="constant coeffcient", type=float)
mutexGroup = argumentParser.add_mutually_exclusive_group()
mutexGroup.add_argument("-q", "--quiet", # here's the new argument
action='store_true',
help="silent output")
mutexGroup.add_argument("-v", "--verbose",
action='store_true',
help="verbose output")
args = argumentParser.parse_args()
discr = args.b**2 - 4*args.a*args.c
root1 = (-args.b + discr**0.5)/(2*args.a)
root2 = (-args.b - discr**0.5)/(2*args.a)
if args.verbose:
print("{:g}x**2{:+g}x{:+g} = (x{:+g})(x{:+g})".format(
args.a, args.b, args.c, -root1, -root2))
elif args.quiet:
pass
else:
print("roots: {}, {}".format(root1, root2))
The "--quiet" flag is nonsensical: No output, so why run the program? Nevertheless, it works:
!python3 demos/d08_argparse_quad_soln_verbose_quiet.py -q 1 -7 12
argparse will enforce the exclusivity:
!python3 demos/d08_argparse_quad_soln_verbose_quiet.py -q -v 1 -7 12
Note the change to the help message:
!python3 demos/d08_argparse_quad_soln_verbose_quiet.py -h
os Module¶The sys module contains system-independent Python internals like sys.argv[] and sys.path. The os module contains support functions and system dependencies from the operating system. Here are some useful attributes:
import os
def printSys(name):
print("{:>15}: {}".format(name, repr(eval(name))))
printSys("os.name")
printSys("os.curdir")
printSys("os.pardir") # parent directory
printSys("os.sep") # pathname separator: "/" on POSIX, "\" on Windows
printSys("os.extsep") # extension separator (always ".")
printSys("os.altsep")
printSys("os.pathsep") # separator in $PATH, etc.
printSys("os.linesep")
printSys("os.defpath") # default path for executables
Path related functions are in the os.path submodule:
dir(os.path)
os.path.exists("xyzzy")
?os.path
The os module itself contains functions that act very much like POSIX (UNIX manual section (2)) functions, with some modifications for Python such as taking the Pythonic form of functions:
outputs = f(inputs)
and not having to pass a buffer as a pointer and length.
?os.open
Remember these are low level I/O routines. Don't use them when normal Python I/O does what you want. Think of normal Python I/O like standard I/O in C or iostreams in C++.
Recall how we do conventional "stream" based I/O. Let's create a temporary file:
%%file temp.txt
This is a file that contains a single line of text.
We read the first character from that file with the read() built-in function:
f = open("temp.txt", "r")
c = f.read(1)
c
Sure enough, it's type is a str because we opened it as a text file:
type(c)
This is because the read() encoded it into the default UTF-8 encoding. But if we open it as a "binary" file, we get a guaranteed 8-bit bytes:
f = open("temp.txt", "rb")
c = f.read(1)
c
type(c)
The os module also allows us to read it with the Python equivalent of the POSIX open(2) system call.
g = os.open("temp.txt", os.O_RDONLY)
r = os.read(g, 1)
r
which is also a bytes because low-level I/O doesn't know anything about encoding.
So os.read() works just like built-in read() of a stream that's open for binary reading.
How about something that stream I/O can't do?
stat(2) Call¶The os module gives us pretty much all the POSIX functions. One of the most important of these is os.stat(), which gives us "metadata" about any filesystem object:
s = os.stat("temp.txt")
s
type(s)
The return of os.stat() is a custom-designed object that can be accessed by attribute:
f'{s.st_mode:o}'
or by index:
s[0], s[1], s[2]
It is also immutable:
s.st_mode = 0
See how this is done?
Among the POSIX routines are the usual fork() .. exec() routines you learned about in CptS 360.
%%file demos/d09_fork_test.py
import os
import sys
import time
def usefulFun(sibId):
pid = os.getpid()
for ct in range(3):
print("In process {} (child {}), the count is {}.".format(
pid, sibId, ct))
sys.stdout.flush() # shows output in more chronological order
time.sleep((sibId+1)*2 + 10)
print('child {} exiting', sibId)
nChild = 3
pList = []
for sibId in range(nChild):
pid = os.fork()
if pid != 0:
# we are in the parent process
pList.append(pid)
else:
# we are in the child process
usefulFun(sibId)
quit()
time.sleep(20) # in parent
print("Process list: ", pList)
print("The parent is exiting")
!python3 demos/d09_fork_test.py
Note that the processes operate asynchronously. This is fine, except that the parent exits before the children. If the children were producing output the parent was using, the parent might want to wait() for them. We can fix that:
%%file demos/d10_fork_wait.py
import os
import sys
import time
def usefulFun(sibId):
for ct in range(2, 0, -1):
print("In process {}, the count is {}.".format(sibId, ct))
time.sleep(sibId)
sys.stdout.flush()
nChild = 3
pList = []
for sibId in range(nChild):
pid = os.fork()
if pid != 0:
# we are in the parent process
pList.append(pid)
else:
# we are in the child process
usefulFun(sibId)
sys.exit()
for sibId in range(nChild):
os.wait()
print("Process list: ", pList)
print("The parent is exiting")
!python3 demos/d10_fork_wait.py
Python supports POSIX threads. Here's a simple application to compute Fibonacci numbers. Each thread takes the last two numbers on the queue, adds them, and appends the result.
Instead of placing a limit on the number of Fibonacci numbers created, we place a time limit on the amount of time used.
%%file demos/d11_thread_fibonacci.py
import threading
import queue
import time
import random
class FiboWorker(threading.Thread):
count = 0
def __init__(self, theQueue):
threading.Thread.__init__(self)
self.theQueue = theQueue
self.delay = random.random()*4
self.id = FiboWorker.count
FiboWorker.count += 1
def run(self):
while 1:
data = self.theQueue.get()
if data == (0,0):
self.theQueue.put(data)
print("at {}, Thread {} exits".format(time.ctime(), self))
break
result = (data[1], data[0]+data[1])
print("at {}, Thread {} computes {} -> {}".format(
time.ctime(), self, data, result))
self.theQueue.put(result)
time.sleep(self.delay) # give other threads a chance
def __str__(self):
return str(self.id)
class FiboWorkerPool:
def __init__(self, nWorkers):
self.theQueue = queue.Queue(10)
# setup pool
self.workerPool = []
for iWorker in range(nWorkers):
self.workerPool.append(FiboWorker(self.theQueue))
def startPool(self):
"""
initializes the queue by putting the initial tuple on it
"""
self.theQueue.put( (0,1) )
for x in self.workerPool:
x.start()
def killPool(self):
"""
terminates the queue by putting (0,0) on it
"""
self.theQueue.put( (0,0) )
if __name__ == "__main__":
thePool = FiboWorkerPool(5)
thePool.startPool()
time.sleep(10)
thePool.killPool()
!python3 demos/d11_thread_fibonacci.py
Here's another example: a threaded simulator of Brownian motion (of molecules):
%%file demos/d12_thread_brownian.py
# Brownian motion -- an example of a multi-threaded Tkinter program.
# developed by Guido van Rossum
from tkinter import *
import random
import threading
import time
import sys
WIDTH = 600
HEIGHT = 600
SIGMA = 10
BUZZ = 2
RADIUS = 4
LAMBDA = 10
GREEN = 'green'
def particle(canvas):
r = RADIUS
x = random.gauss(WIDTH/2.0, SIGMA)
y = random.gauss(HEIGHT/2.0, SIGMA)
p = canvas.create_oval(x-r, y-r, x+r, y+r, fill=GREEN)
while True:
dx = random.gauss(0, BUZZ)
dy = random.gauss(0, BUZZ)
dt = random.expovariate(LAMBDA)
try:
canvas.move(p, dx, dy)
except RuntimeError:
break
time.sleep(dt)
def bye(event, widget):
threading.abort()
widget.quit()
def main():
root = Tk()
canvas = Canvas(root, width=WIDTH, height=HEIGHT)
canvas.bind('<Destroy>', lambda event, widget=canvas: bye)
canvas.grid()
np = 10
if sys.argv[1:]:
np = int(sys.argv[1])
for n in range(np):
t = threading.Thread(target=particle, args=(canvas,))
try:
t.start()
except:
print("""
unable to create {} threads -- continuing with {} threads
"""[1:-1].format(np, n))
break
root.mainloop()
if __name__ == '__main__':
main()
Start this in another shell window:
top -H -u $USER
!python3 demos/d12_thread_brownian.py 1000