From 5ec5376f5ba52c102981c8e66c07e8c33e8436e3 Mon Sep 17 00:00:00 2001 From: yuchen Date: Sun, 15 Mar 2015 16:54:19 +0800 Subject: [PATCH] add ostep homeworks --- related_info/ostep/README.md | 33 + related_info/ostep/ostep1-relocation.md | 98 ++ related_info/ostep/ostep1-relocation.py | 117 ++ related_info/ostep/ostep10-lottery.md | 124 ++ related_info/ostep/ostep10-lottery.py | 119 ++ related_info/ostep/ostep11-threadintro/loop.s | 6 + .../ostep11-threadintro/looping-race-nolock.s | 15 + .../ostep/ostep11-threadintro/race.md | 329 +++++ .../ostep/ostep11-threadintro/simple-race.s | 6 + .../ostep/ostep11-threadintro/wait-for-me.s | 13 + related_info/ostep/ostep11-threadintro/x86.py | 989 ++++++++++++++ related_info/ostep/ostep12-threadlock/flag.s | 27 + .../ostep/ostep12-threadlock/locks.md | 351 +++++ .../ostep/ostep12-threadlock/peterson.s | 53 + .../ostep/ostep12-threadlock/test-and-set.s | 26 + .../test-and-test-and-set.s | 29 + .../ostep/ostep12-threadlock/ticket.s | 30 + related_info/ostep/ostep12-threadlock/x86.py | 1186 +++++++++++++++++ related_info/ostep/ostep12-threadlock/yield.s | 29 + related_info/ostep/ostep13-vsfs.md | 256 ++++ related_info/ostep/ostep13-vsfs.py | 551 ++++++++ related_info/ostep/ostep14-afs.md | 252 ++++ related_info/ostep/ostep14-afs.py | 600 +++++++++ .../ostep/ostep15-disk/disk-precise.py | 719 ++++++++++ related_info/ostep/ostep15-disk/disk.md | 133 ++ related_info/ostep/ostep15-disk/disk.py | 730 ++++++++++ related_info/ostep/ostep16-raid.md | 223 ++++ related_info/ostep/ostep16-raid.py | 447 +++++++ related_info/ostep/ostep2-segmentation.md | 166 +++ related_info/ostep/ostep2-segmentation.py | 193 +++ related_info/ostep/ostep3-malloc.md | 162 +++ related_info/ostep/ostep3-malloc.py | 238 ++++ .../ostep/ostep4-paging-linear-translate.md | 131 ++ .../ostep/ostep4-paging-linear-translate.py | 192 +++ .../ostep5-paging-multilevel-translate.md | 71 + .../ostep5-paging-multilevel-translate.py | 263 ++++ related_info/ostep/ostep6-paging-policy.md | 119 ++ related_info/ostep/ostep6-paging-policy.py | 275 ++++ related_info/ostep/ostep7-process-run.md | 213 +++ related_info/ostep/ostep7-process-run.py | 325 +++++ related_info/ostep/ostep8-scheduler.md | 116 ++ related_info/ostep/ostep8-scheduler.py | 155 +++ related_info/ostep/ostep9-mlfq.md | 174 +++ related_info/ostep/ostep9-mlfq.py | 326 +++++ 44 files changed, 10610 insertions(+) create mode 100644 related_info/ostep/README.md create mode 100644 related_info/ostep/ostep1-relocation.md create mode 100644 related_info/ostep/ostep1-relocation.py create mode 100644 related_info/ostep/ostep10-lottery.md create mode 100755 related_info/ostep/ostep10-lottery.py create mode 100644 related_info/ostep/ostep11-threadintro/loop.s create mode 100644 related_info/ostep/ostep11-threadintro/looping-race-nolock.s create mode 100644 related_info/ostep/ostep11-threadintro/race.md create mode 100644 related_info/ostep/ostep11-threadintro/simple-race.s create mode 100644 related_info/ostep/ostep11-threadintro/wait-for-me.s create mode 100755 related_info/ostep/ostep11-threadintro/x86.py create mode 100644 related_info/ostep/ostep12-threadlock/flag.s create mode 100644 related_info/ostep/ostep12-threadlock/locks.md create mode 100644 related_info/ostep/ostep12-threadlock/peterson.s create mode 100644 related_info/ostep/ostep12-threadlock/test-and-set.s create mode 100644 related_info/ostep/ostep12-threadlock/test-and-test-and-set.s create mode 100644 related_info/ostep/ostep12-threadlock/ticket.s create mode 100755 related_info/ostep/ostep12-threadlock/x86.py create mode 100644 related_info/ostep/ostep12-threadlock/yield.s create mode 100644 related_info/ostep/ostep13-vsfs.md create mode 100755 related_info/ostep/ostep13-vsfs.py create mode 100644 related_info/ostep/ostep14-afs.md create mode 100644 related_info/ostep/ostep14-afs.py create mode 100755 related_info/ostep/ostep15-disk/disk-precise.py create mode 100644 related_info/ostep/ostep15-disk/disk.md create mode 100755 related_info/ostep/ostep15-disk/disk.py create mode 100644 related_info/ostep/ostep16-raid.md create mode 100755 related_info/ostep/ostep16-raid.py create mode 100644 related_info/ostep/ostep2-segmentation.md create mode 100755 related_info/ostep/ostep2-segmentation.py create mode 100644 related_info/ostep/ostep3-malloc.md create mode 100755 related_info/ostep/ostep3-malloc.py create mode 100644 related_info/ostep/ostep4-paging-linear-translate.md create mode 100755 related_info/ostep/ostep4-paging-linear-translate.py create mode 100644 related_info/ostep/ostep5-paging-multilevel-translate.md create mode 100755 related_info/ostep/ostep5-paging-multilevel-translate.py create mode 100644 related_info/ostep/ostep6-paging-policy.md create mode 100755 related_info/ostep/ostep6-paging-policy.py create mode 100644 related_info/ostep/ostep7-process-run.md create mode 100755 related_info/ostep/ostep7-process-run.py create mode 100644 related_info/ostep/ostep8-scheduler.md create mode 100755 related_info/ostep/ostep8-scheduler.py create mode 100644 related_info/ostep/ostep9-mlfq.md create mode 100755 related_info/ostep/ostep9-mlfq.py diff --git a/related_info/ostep/README.md b/related_info/ostep/README.md new file mode 100644 index 0000000..2b3dce2 --- /dev/null +++ b/related_info/ostep/README.md @@ -0,0 +1,33 @@ +# README +These resources are from [Operating Systems: Three Easy Pieces](http://pages.cs.wisc.edu/~remzi/OSTEP/) by Remzi H. Arpaci-Dusseau and Andrea C. Arpaci-Dusseau + +We use&modify them as some homeworks of our OS course. + +## Memory + - ostep1-relocation.md + - ostep2-segmentation.md + - ostep3-malloc.md + - ostep4-paging-linear-translate.md + - ostep5-paging-multilevel-translate.md + - ostep6-paging-policy.md + +## process + - ostep7-process-run.md + +## scheduling + - ostep8-scheduler.md + - ostep9-mlfq.md + - ostep10-lottery.md + +## sync/mutex + - ostep11-threadintro/race.md + - ostep12-threadlock/locks.md + +## file system + - ostep13-vsfs.md + - ostep14-afs.md + +## disk + - ostep15-disk/disk.md + - ostep16-raid.md + diff --git a/related_info/ostep/ostep1-relocation.md b/related_info/ostep/ostep1-relocation.md new file mode 100644 index 0000000..a118e38 --- /dev/null +++ b/related_info/ostep/ostep1-relocation.md @@ -0,0 +1,98 @@ + +This program allows you to see how address translations are performed in a +system with base and bounds registers. As before, there are two steps to +running the program to test out your understanding of base and bounds. First, +run without the -c flag to generate a set of translations and see if you can +correctly perform the address translations yourself. Then, when done, run with +the -c flag to check your answers. + +In this homework, we will assume a slightly different address space than our +canonical one with a heap and stack at opposite ends of the space. Rather, we +will assume that the address space has a code section, then a fixed-sized +(small) stack, and a heap that grows downward right after, looking something +like you see in the Figure below. In this configuration, there is only one +direction of growth, towards higher regions of the address space. + + -------------- 0KB + | Code | + -------------- 2KB + | Stack | + -------------- 4KB + | Heap | + | | | + | v | + -------------- 7KB + | (free) | + | ... | + +In the figure, the bounds register would be set to 7~KB, as that represents +the end of the address space. References to any address within the bounds +would be considered legal; references above this value are out of bounds and +thus the hardware would raise an exception. + +To run with the default flags, type relocation.py at the command line. The +result should be something like this: + +prompt> ./relocation.py +... +Base-and-Bounds register information: + + Base : 0x00003082 (decimal 12418) + Limit : 472 + +Virtual Address Trace + VA 0: 0x01ae (decimal:430) -> PA or violation? + VA 1: 0x0109 (decimal:265) -> PA or violation? + VA 2: 0x020b (decimal:523) -> PA or violation? + VA 3: 0x019e (decimal:414) -> PA or violation? + VA 4: 0x0322 (decimal:802) -> PA or violation? + +For each virtual address, either write down the physical address it +translates to OR write down that it is an out-of-bounds address +(a segmentation violation). For this problem, you should assume a +simple virtual address space of a given size. + +As you can see, the homework simply generates randomized virtual +addresses. For each, you should determine whether it is in bounds, and if so, +determine to which physical address it translates. Running with -c (the +"compute this for me" flag) gives us the results of these translations, i.e., +whether they are valid or not, and if valid, the resulting physical +addresses. For convenience, all numbers are given both in hex and decimal. + +prompt> ./relocation.py -c +... +Virtual Address Trace + VA 0: 0x01ae (decimal:430) -> VALID: 0x00003230 (dec:12848) + VA 1: 0x0109 (decimal:265) -> VALID: 0x0000318b (dec:12683) + VA 2: 0x020b (decimal:523) -> SEGMENTATION VIOLATION + VA 3: 0x019e (decimal:414) -> VALID: 0x00003220 (dec:12832) + VA 4: 0x0322 (decimal:802) -> SEGMENTATION VIOLATION +] + +With a base address of 12418 (decimal), address 430 is within bounds (i.e., it +is less than the limit register of 472) and thus translates to 430 added to +12418 or 12848. A few of the addresses shown above are out of bounds (523, +802), as they are in excess of the bounds. Pretty simple, no? Indeed, that is +one of the beauties of base and bounds: it's so darn simple! + +There are a few flags you can use to control what's going on better: + +prompt> ./relocation.py -h +Usage: relocation.py [options] + +Options: + -h, --help show this help message and exit + -s SEED, --seed=SEED the random seed + -a ASIZE, --asize=ASIZE address space size (e.g., 16, 64k, 32m) + -p PSIZE, --physmem=PSIZE physical memory size (e.g., 16, 64k) + -n NUM, --addresses=NUM # of virtual addresses to generate + -b BASE, --b=BASE value of base register + -l LIMIT, --l=LIMIT value of limit register + -c, --compute compute answers for me +] + +In particular, you can control the virtual address-space size (-a), the size +of physical memory (-p), the number of virtual addresses to generate (-n), and +the values of the base and bounds registers for this process (-b and -l, +respectively). + diff --git a/related_info/ostep/ostep1-relocation.py b/related_info/ostep/ostep1-relocation.py new file mode 100644 index 0000000..8ec4abb --- /dev/null +++ b/related_info/ostep/ostep1-relocation.py @@ -0,0 +1,117 @@ +#! /usr/bin/env python + +import sys +from optparse import OptionParser +import random +import math + +def convert(size): + length = len(size) + lastchar = size[length-1] + if (lastchar == 'k') or (lastchar == 'K'): + m = 1024 + nsize = int(size[0:length-1]) * m + elif (lastchar == 'm') or (lastchar == 'M'): + m = 1024*1024 + nsize = int(size[0:length-1]) * m + elif (lastchar == 'g') or (lastchar == 'G'): + m = 1024*1024*1024 + nsize = int(size[0:length-1]) * m + else: + nsize = int(size) + return nsize + + +# +# main program +# +parser = OptionParser() +parser.add_option('-s', '--seed', default=0, help='the random seed', action='store', type='int', dest='seed') +parser.add_option('-a', '--asize', default='1k', help='address space size (e.g., 16, 64k, 32m, 1g)', action='store', type='string', dest='asize') +parser.add_option('-p', '--physmem', default='16k', help='physical memory size (e.g., 16, 64k, 32m, 1g)', action='store', type='string', dest='psize') +parser.add_option('-n', '--addresses', default=5, help='number of virtual addresses to generate', action='store', type='int', dest='num') +parser.add_option('-b', '--b', default='-1', help='value of base register', action='store', type='string', dest='base') +parser.add_option('-l', '--l', default='-1', help='value of limit register', action='store', type='string', dest='limit') +parser.add_option('-c', '--compute', default=False, help='compute answers for me', action='store_true', dest='solve') + + +(options, args) = parser.parse_args() + +print '' +print 'ARG seed', options.seed +print 'ARG address space size', options.asize +print 'ARG phys mem size', options.psize +print '' + +random.seed(options.seed) +asize = convert(options.asize) +psize = convert(options.psize) + +if psize <= 1: + print 'Error: must specify a non-zero physical memory size.' + exit(1) + +if asize == 0: + print 'Error: must specify a non-zero address-space size.' + exit(1) + +if psize <= asize: + print 'Error: physical memory size must be GREATER than address space size (for this simulation)' + exit(1) + +# +# need to generate base, bounds for segment registers +# +limit = convert(options.limit) +base = convert(options.base) + +if limit == -1: + limit = int(asize/4.0 + (asize/4.0 * random.random())) + +# now have to find room for them +if base == -1: + done = 0 + while done == 0: + base = int(psize * random.random()) + if (base + limit) < psize: + done = 1 + +print 'Base-and-Bounds register information:' +print '' +print ' Base : 0x%08x (decimal %d)' % (base, base) +print ' Limit : %d' % (limit) +print '' + +if base + limit > psize: + print 'Error: address space does not fit into physical memory with those base/bounds values.' + print 'Base + Limit:', base + limit, ' Psize:', psize + exit(1) + +# +# now, need to generate virtual address trace +# +print 'Virtual Address Trace' +for i in range(0,options.num): + vaddr = int(asize * random.random()) + if options.solve == False: + print ' VA %2d: 0x%08x (decimal: %4d) --> PA or segmentation violation?' % (i, vaddr, vaddr) + else: + paddr = 0 + if (vaddr >= limit): + print ' VA %2d: 0x%08x (decimal: %4d) --> SEGMENTATION VIOLATION' % (i, vaddr, vaddr) + else: + paddr = vaddr + base + print ' VA %2d: 0x%08x (decimal: %4d) --> VALID: 0x%08x (decimal: %4d)' % (i, vaddr, vaddr, paddr, paddr) + + +print '' + +if options.solve == False: + print 'For each virtual address, either write down the physical address it translates to' + print 'OR write down that it is an out-of-bounds address (a segmentation violation). For' + print 'this problem, you should assume a simple virtual address space of a given size.' + print '' + + + + diff --git a/related_info/ostep/ostep10-lottery.md b/related_info/ostep/ostep10-lottery.md new file mode 100644 index 0000000..5e75c09 --- /dev/null +++ b/related_info/ostep/ostep10-lottery.md @@ -0,0 +1,124 @@ + +This program, lottery.py, allows you to see how a lottery scheduler +works. As always, there are two steps to running the program. First, run +without the -c flag: this shows you what problem to solve without +revealing the answers. + +prompt> ./lottery.py -j 2 -s 0 +... +Here is the job list, with the run time of each job: + Job 0 ( length = 8, tickets = 75 ) + Job 1 ( length = 4, tickets = 25 ) + +Here is the set of random numbers you will need (at most): +Random 511275 +Random 404934 +Random 783799 +Random 303313 +Random 476597 +Random 583382 +Random 908113 +Random 504687 +Random 281838 +Random 755804 +Random 618369 +Random 250506 +] + +When you run the simulator in this manner, it first assigns you some random +jobs (here of lengths 8, and 4), each with some number of tickets (here 75 and +25, respectively). The simulator also gives you a list of random numbers, +which you will need to determine what the lottery scheduler will do. The +random numbers are chosen to be between 0 and a large number; thus, you'll +have to use the modulo operator to compute the lottery winner (i.e., winner +should equal this random number modulo the total number of tickets in the +system). + +Running with -c shows exactly what you are supposed to calculate: + +prompt> ./lottery.py -j 2 -s 0 -c +... +** Solutions ** +Random 511275 -> Winning ticket 75 (of 100) -> Run 1 + Jobs: ( job:0 timeleft:8 tix:75 ) (* job:1 timeleft:4 tix:25 ) +Random 404934 -> Winning ticket 34 (of 100) -> Run 0 + Jobs: (* job:0 timeleft:8 tix:75 ) ( job:1 timeleft:3 tix:25 ) +Random 783799 -> Winning ticket 99 (of 100) -> Run 1 + Jobs: ( job:0 timeleft:7 tix:75 ) (* job:1 timeleft:3 tix:25 ) +Random 303313 -> Winning ticket 13 (of 100) -> Run 0 + Jobs: (* job:0 timeleft:7 tix:75 ) ( job:1 timeleft:2 tix:25 ) +Random 476597 -> Winning ticket 97 (of 100) -> Run 1 + Jobs: ( job:0 timeleft:6 tix:75 ) (* job:1 timeleft:2 tix:25 ) +Random 583382 -> Winning ticket 82 (of 100) -> Run 1 + Jobs: ( job:0 timeleft:6 tix:75 ) (* job:1 timeleft:1 tix:25 ) +--> JOB 1 DONE at time 6 +Random 908113 -> Winning ticket 13 (of 75) -> Run 0 + Jobs: (* job:0 timeleft:6 tix:75 ) ( job:1 timeleft:0 tix:--- ) +Random 504687 -> Winning ticket 12 (of 75) -> Run 0 + Jobs: (* job:0 timeleft:5 tix:75 ) ( job:1 timeleft:0 tix:--- ) +Random 281838 -> Winning ticket 63 (of 75) -> Run 0 + Jobs: (* job:0 timeleft:4 tix:75 ) ( job:1 timeleft:0 tix:--- ) +Random 755804 -> Winning ticket 29 (of 75) -> Run 0 + Jobs: (* job:0 timeleft:3 tix:75 ) ( job:1 timeleft:0 tix:--- ) +Random 618369 -> Winning ticket 69 (of 75) -> Run 0 + Jobs: (* job:0 timeleft:2 tix:75 ) ( job:1 timeleft:0 tix:--- ) +Random 250506 -> Winning ticket 6 (of 75) -> Run 0 + Jobs: (* job:0 timeleft:1 tix:75 ) ( job:1 timeleft:0 tix:--- ) +--> JOB 0 DONE at time 12 +] + +As you can see from this trace, what you are supposed to do is use the random +number to figure out which ticket is the winner. Then, given the winning +ticket, figure out which job should run. Repeat this until all of the jobs are +finished running. It's as simple as that -- you are just emulating what the +lottery scheduler does, but by hand! + +Just to make this absolutely clear, let's look at the first decision made in +the example above. At this point, we have two jobs (job 0 which has a runtime +of 8 and 75 tickets, and job 1 which has a runtime of 4 and 25 tickets). The +first random number we are given is 511275. As there are 100 tickets in the +system, 511275 \% 100 is 75, and thus 75 is our winning ticket. + +If ticket 75 is the winner, we simply search through the job list until we +find it. The first entry, for job 0, has 75 tickets (0 through 74), and thus +does not win; the next entry is for job 1, and thus we have found our winner, +so we run job 1 for the quantum length (1 in this example). All of this is +shown in the print out as follows: + +Random 511275 -> Winning ticket 75 (of 100) -> Run 1 + Jobs: ( job:0 timeleft:8 tix:75 ) (* job:1 timeleft:4 tix:25 ) +] + +As you can see, the first line summarizes what happens, and the second simply +shows the entire job queue, with an * denoting which job was chosen. + +The simulator has a few other options, most of which should be +self-explanatory. Most notably, the -l/--jlist flag can be used to specify an +exact set of jobs and their ticket values, instead of always using +randomly-generated job lists. + +prompt> ./lottery.py -h +Usage: lottery.py [options] + +Options: + -h, --help + show this help message and exit + -s SEED, --seed=SEED + the random seed + -j JOBS, --jobs=JOBS + number of jobs in the system + -l JLIST, --jlist=JLIST + instead of random jobs, provide a comma-separated list + of run times and ticket values (e.g., 10:100,20:100 + would have two jobs with run-times of 10 and 20, each + with 100 tickets) + -m MAXLEN, --maxlen=MAXLEN + max length of job + -T MAXTICKET, --maxtick=MAXTICKET + maximum ticket value, if randomly assigned + -q QUANTUM, --quantum=QUANTUM + length of time slice + -c, --compute + compute answers for me + + diff --git a/related_info/ostep/ostep10-lottery.py b/related_info/ostep/ostep10-lottery.py new file mode 100755 index 0000000..b0fd6be --- /dev/null +++ b/related_info/ostep/ostep10-lottery.py @@ -0,0 +1,119 @@ +#! /usr/bin/env python + +import sys +from optparse import OptionParser +import random + +parser = OptionParser() +parser.add_option('-s', '--seed', default=0, help='the random seed', action='store', type='int', dest='seed') +parser.add_option('-j', '--jobs', default=3, help='number of jobs in the system', action='store', type='int', dest='jobs') +parser.add_option('-l', '--jlist', default='', help='instead of random jobs, provide a comma-separated list of run times and ticket values (e.g., 10:100,20:100 would have two jobs with run-times of 10 and 20, each with 100 tickets)', action='store', type='string', dest='jlist') +parser.add_option('-m', '--maxlen', default=10, help='max length of job', action='store', type='int', dest='maxlen') +parser.add_option('-T', '--maxticket', default=100, help='maximum ticket value, if randomly assigned', action='store', type='int', dest='maxticket') +parser.add_option('-q', '--quantum', default=1, help='length of time slice', action='store', type='int', dest='quantum') +parser.add_option('-c', '--compute', help='compute answers for me', action='store_true', default=False, dest='solve') + +(options, args) = parser.parse_args() + +random.seed(options.seed) + +print 'ARG jlist', options.jlist +print 'ARG jobs', options.jobs +print 'ARG maxlen', options.maxlen +print 'ARG maxticket', options.maxticket +print 'ARG quantum', options.quantum +print 'ARG seed', options.seed +print '' + +print 'Here is the job list, with the run time of each job: ' + +import operator + + +tickTotal = 0 +runTotal = 0 +joblist = [] +if options.jlist == '': + for jobnum in range(0,options.jobs): + runtime = int(options.maxlen * random.random()) + tickets = int(options.maxticket * random.random()) + runTotal += runtime + tickTotal += tickets + joblist.append([jobnum, runtime, tickets]) + print ' Job %d ( length = %d, tickets = %d )' % (jobnum, runtime, tickets) +else: + jobnum = 0 + for entry in options.jlist.split(','): + (runtime, tickets) = entry.split(':') + joblist.append([jobnum, int(runtime), int(tickets)]) + runTotal += int(runtime) + tickTotal += int(tickets) + jobnum += 1 + for job in joblist: + print ' Job %d ( length = %d, tickets = %d )' % (job[0], job[1], job[2]) +print '\n' + +if options.solve == False: + print 'Here is the set of random numbers you will need (at most):' + for i in range(runTotal): + r = int(random.random() * 1000001) + print 'Random', r + +if options.solve == True: + print '** Solutions **\n' + + jobs = len(joblist) + clock = 0 + for i in range(runTotal): + r = int(random.random() * 1000001) + winner = int(r % tickTotal) + + current = 0 + for (job, runtime, tickets) in joblist: + current += tickets + if current > winner: + (wjob, wrun, wtix) = (job, runtime, tickets) + break + + print 'Random', r, '-> Winning ticket %d (of %d) -> Run %d' % (winner, tickTotal, wjob) + # print 'Winning ticket %d (of %d) -> Run %d' % (winner, tickTotal, wjob) + + print ' Jobs:', + for (job, runtime, tickets) in joblist: + if wjob == job: + wstr = '*' + else: + wstr = ' ' + + if runtime > 0: + tstr = tickets + else: + tstr = '---' + print ' (%s job:%d timeleft:%d tix:%s ) ' % (wstr, job, runtime, tstr), + print '' + + # now do the accounting + if wrun >= options.quantum: + wrun -= options.quantum + else: + wrun = 0 + + clock += options.quantum + + # job completed! + if wrun == 0: + print '--> JOB %d DONE at time %d' % (wjob, clock) + tickTotal -= wtix + wtix = 0 + jobs -= 1 + + # update job list + joblist[wjob] = (wjob, wrun, wtix) + + if jobs == 0: + print '' + break + + + + diff --git a/related_info/ostep/ostep11-threadintro/loop.s b/related_info/ostep/ostep11-threadintro/loop.s new file mode 100644 index 0000000..6ff0dc8 --- /dev/null +++ b/related_info/ostep/ostep11-threadintro/loop.s @@ -0,0 +1,6 @@ +.main +.top +sub $1,%dx +test $0,%dx +jgte .top +halt diff --git a/related_info/ostep/ostep11-threadintro/looping-race-nolock.s b/related_info/ostep/ostep11-threadintro/looping-race-nolock.s new file mode 100644 index 0000000..4508837 --- /dev/null +++ b/related_info/ostep/ostep11-threadintro/looping-race-nolock.s @@ -0,0 +1,15 @@ +# assumes %bx has loop count in it + +.main +.top +# critical section +mov 2000, %ax # get the value at the address +add $1, %ax # increment it +mov %ax, 2000 # store it back + +# see if we're still looping +sub $1, %bx +test $0, %bx +jgt .top + +halt diff --git a/related_info/ostep/ostep11-threadintro/race.md b/related_info/ostep/ostep11-threadintro/race.md new file mode 100644 index 0000000..dd75bc7 --- /dev/null +++ b/related_info/ostep/ostep11-threadintro/race.md @@ -0,0 +1,329 @@ + +Welcome to this simulator. The idea is to gain familiarity with threads by +seeing how they interleave; the simulator, x86.py, will help you in +gaining this understanding. + +The simulator mimicks the execution of short assembly sequences by multiple +threads. Note that the OS code that would run (for example, to perform a +context switch) is *not* shown; thus, all you see is the interleaving of the +user code. + +The assembly code that is run is based on x86, but somewhat simplified. +In this instruction set, there are four general-purpose registers +(%ax, %bx, %cx, %dx), a program counter (PC), and a small set of instructions +which will be enough for our purposes. + +Here is an example code snippet that we will be able to run: + +.main +mov 2000, %ax # get the value at the address +add $1, %ax # increment it +mov %ax, 2000 # store it back +halt + +The code is easy to understand. The first instruction, an x86 "mov", simply +loads a value from the address specified by 2000 into the register %ax. +Addresses, in this subset of x86, can take some of the following forms: + + 2000 -> the number (2000) is the address + (%cx) -> contents of register (in parentheses) forms the address + 1000(%dx) -> the number + contents of the register form the address + 10(%ax,%bx) -> the number + reg1 + reg2 forms the address + +To store a value, the same "mov" instruction is used, but this time with the +arguments reversed, e.g.: + + mov %ax, 2000 + +The "add" instruction, from the sequence above, should be clear: it adds an +immediate value (specified by $1) to the register specified in the second +argument (i.e., %ax = %ax + 1). + +Thus, we now can understand the code sequence above: it loads the value at +address 2000, adds 1 to it, and then stores the value back into address 2000. + +The fake-ish "halt" instruction just stops running this thread. + +Let's run the simulator and see how this all works! Assume the above code +sequence is in the file "simple-race.s". + +prompt> ./x86.py -p simple-race.s -t 1 + + Thread 0 +1000 mov 2000, %ax +1001 add $1, %ax +1002 mov %ax, 2000 +1003 halt + +prompt> + +The arguments used here specify the program (-p), the number of threads (-t +1), and the interrupt interval, which is how often a scheduler will be woken +and run to switch to a different task. Because there is only one thread in +this example, this interval does not matter. + +The output is easy to read: the simulator prints the program counter (here +shown from 1000 to 1003) and the instruction that gets executed. Note that we +assume (unrealistically) that all instructions just take up a single byte in +memory; in x86, instructions are variable-sized and would take up from one to +a small number of bytes. + +We can use more detailed tracing to get a better sense of how machine state +changes during the execution: + +prompt> ./x86.py -p simple-race.s -t 1 -M 2000 -R ax,bx + + 2000 ax bx Thread 0 + ? ? ? + ? ? ? 1000 mov 2000, %ax + ? ? ? 1001 add $1, %ax + ? ? ? 1002 mov %ax, 2000 + ? ? ? 1003 halt + +Oops! Forgot the -c flag (which actually computes the answers for you). + +prompt> ./x86.py -p simple-race.s -t 1 -M 2000 -R ax,bx -c + + 2000 ax bx Thread 0 + 0 0 0 + 0 0 0 1000 mov 2000, %ax + 0 1 0 1001 add $1, %ax + 1 1 0 1002 mov %ax, 2000 + 1 1 0 1003 halt + +By using the -M flag, we can trace memory locations (a comma-separated list +lets you trace more than one, e.g., 2000,3000); by using the -R flag we can +track the values inside specific registers. + +The values on the left show the memory/register contents AFTER the instruction +on the right has executed. For example, after the "add" instruction, you can +see that %ax has been incremented to the value 1; after the second "mov" +instruction (at PC=1002), you can see that the memory contents at 2000 are +now also incremented. + +There are a few more instructions you'll need to know, so let's get to them +now. Here is a code snippet of a loop: + +.main +.top +sub $1,%dx +test $0,%dx +jgte .top +halt + +A few things have been introduced here. First is the "test" instruction. +This instruction takes two arguments and compares them; it then sets implicit +"condition codes" (kind of like 1-bit registers) which subsequent instructions +can act upon. + +In this case, the other new instruction is the "jump" instruction (in this +case, "jgte" which stands for "jump if greater than or equal to"). This +instruction jumps if the first value is greater than or equal to the second +in the test. + +One last point: to really make this code work, dx must be initialized to 1 or +greater. + +Thus, we run the program like this: + +prompt> ./x86.py -p loop.s -t 1 -a dx=3 -R dx -C -c + + dx >= > <= < != == Thread 0 + 3 0 0 0 0 0 0 + 2 0 0 0 0 0 0 1000 sub $1,%dx + 2 1 1 0 0 1 0 1001 test $0,%dx + 2 1 1 0 0 1 0 1002 jgte .top + 1 1 1 0 0 1 0 1000 sub $1,%dx + 1 1 1 0 0 1 0 1001 test $0,%dx + 1 1 1 0 0 1 0 1002 jgte .top + 0 1 1 0 0 1 0 1000 sub $1,%dx + 0 1 0 1 0 0 1 1001 test $0,%dx + 0 1 0 1 0 0 1 1002 jgte .top + 0 1 0 1 0 0 1 1003 halt + +The "-R dx" flag traces the value of %dx; the "-C" flag traces the values of +the condition codes that get set by a test instruction. Finally, the "-a dx=3" +flag sets the %dx register to the value 3 to start with. + +As you can see from the trace, the "sub" instruction slowly lowers the value +of %dx. The first few times "test" is called, only the ">=", ">", and "!=" +conditions get set. However, the last "test" in the trace finds %dx and 0 to +be equal, and thus the subsequent jump does NOT take place, and the program +finally halts. + +Now, finally, we get to a more interesting case, i.e., a race condition with +multiple threads. Let's look at the code first: + +.main +.top +# critical section +mov 2000, %ax # get the value at the address +add $1, %ax # increment it +mov %ax, 2000 # store it back + +# see if we're still looping +sub $1, %bx +test $0, %bx +jgt .top + +halt + +The code has a critical section which loads the value of a variable +(at address 2000), then adds 1 to the value, then stores it back. + +The code after just decrements a loop counter (in %bx), tests if it +is greater than or equal to zero, and if so, jumps back to the top +to the critical section again. + +prompt> ./x86.py -p looping-race-nolock.s -t 2 -a bx=1 -M 2000 -c + + 2000 bx Thread 0 Thread 1 + 0 1 + 0 1 1000 mov 2000, %ax + 0 1 1001 add $1, %ax + 1 1 1002 mov %ax, 2000 + 1 0 1003 sub $1, %bx + 1 0 1004 test $0, %bx + 1 0 1005 jgt .top + 1 0 1006 halt + 1 1 ----- Halt;Switch ----- ----- Halt;Switch ----- + 1 1 1000 mov 2000, %ax + 1 1 1001 add $1, %ax + 2 1 1002 mov %ax, 2000 + 2 0 1003 sub $1, %bx + 2 0 1004 test $0, %bx + 2 0 1005 jgt .top + 2 0 1006 halt + +Here you can see each thread ran once, and each updated the shared +variable at address 2000 once, thus resulting in a count of two there. + +The "Halt;Switch" line is inserted whenever a thread halts and another +thread must be run. + +One last example: run the same thing above, but with a smaller interrupt +frequency. Here is what that will look like: + +[mac Race-Analyze] ./x86.py -p looping-race-nolock.s -t 2 -a bx=1 -M 2000 -i 2 + + 2000 Thread 0 Thread 1 + ? + ? 1000 mov 2000, %ax + ? 1001 add $1, %ax + ? ------ Interrupt ------ ------ Interrupt ------ + ? 1000 mov 2000, %ax + ? 1001 add $1, %ax + ? ------ Interrupt ------ ------ Interrupt ------ + ? 1002 mov %ax, 2000 + ? 1003 sub $1, %bx + ? ------ Interrupt ------ ------ Interrupt ------ + ? 1002 mov %ax, 2000 + ? 1003 sub $1, %bx + ? ------ Interrupt ------ ------ Interrupt ------ + ? 1004 test $0, %bx + ? 1005 jgt .top + ? ------ Interrupt ------ ------ Interrupt ------ + ? 1004 test $0, %bx + ? 1005 jgt .top + ? ------ Interrupt ------ ------ Interrupt ------ + ? 1006 halt + ? ----- Halt;Switch ----- ----- Halt;Switch ----- + ? 1006 halt + +As you can see, each thread is interrupt every 2 instructions, as we specify +via the "-i 2" flag. What is the value of memory[2000] throughout this run? +What should it have been? + +Now let's give a little more information on what can be simulated +with this program. The full set of registers: %ax, %bx, %cx, %dx, and the PC. +In this version, there is no support for a "stack", nor are there call +and return instructions. + +The full set of instructions simulated are: + +mov immediate, register # moves immediate value to register +mov memory, register # loads from memory into register +mov register, register # moves value from one register to other +mov register, memory # stores register contents in memory +mov immediate, memory # stores immediate value in memory + +add immediate, register # register = register + immediate +add register1, register2 # register2 = register2 + register1 +sub immediate, register # register = register - immediate +sub register1, register2 # register2 = register2 - register1 + +test immediate, register # compare immediate and register (set condition codes) +test register, immediate # same but register and immediate +test register, register # same but register and register + +jne # jump if test'd values are not equal +je # ... equal +jlt # ... second is less than first +jlte # ... less than or equal +jgt # ... is greater than +jgte # ... greater than or equal + +xchg register, memory # atomic exchange: + # put value of register into memory + # return old contents of memory into reg + # do both things atomically + +nop # no op + +Notes: +- 'immediate' is something of the form $number +- 'memory' is of the form 'number' or '(reg)' or 'number(reg)' or + 'number(reg,reg)' (as described above) +- 'register' is one of %ax, %bx, %cx, %dx + +Finally, here are the full set of options to the simulator are available with +the -h flag: + +Usage: x86.py [options] + +Options: + -h, --help show this help message and exit + -s SEED, --seed=SEED the random seed + -t NUMTHREADS, --threads=NUMTHREADS + number of threads + -p PROGFILE, --program=PROGFILE + source program (in .s) + -i INTFREQ, --interrupt=INTFREQ + interrupt frequency + -r, --randints if interrupts are random + -a ARGV, --argv=ARGV comma-separated per-thread args (e.g., ax=1,ax=2 sets + thread 0 ax reg to 1 and thread 1 ax reg to 2); + specify multiple regs per thread via colon-separated + list (e.g., ax=1:bx=2,cx=3 sets thread 0 ax and bx and + just cx for thread 1) + -L LOADADDR, --loadaddr=LOADADDR + address where to load code + -m MEMSIZE, --memsize=MEMSIZE + size of address space (KB) + -M MEMTRACE, --memtrace=MEMTRACE + comma-separated list of addrs to trace (e.g., + 20000,20001) + -R REGTRACE, --regtrace=REGTRACE + comma-separated list of regs to trace (e.g., + ax,bx,cx,dx) + -C, --cctrace should we trace condition codes + -S, --printstats print some extra stats + -c, --compute compute answers for me + + +Most are obvious. Usage of -r turns on a random interrupter (from 1 to intfreq +as specified by -i), which can make for more fun during homework problems. + +-L specifies where in the address space to load the code. + +-m specified the size of the address space (in KB). + +-S prints some extra stats + +-c is not really used (unlike most simulators in the book); use the tracing + +or condition codes. + +Now you have the basics in place; read the questions at the end of the chapter +to study this race condition and related issues in more depth. + diff --git a/related_info/ostep/ostep11-threadintro/simple-race.s b/related_info/ostep/ostep11-threadintro/simple-race.s new file mode 100644 index 0000000..f3b329e --- /dev/null +++ b/related_info/ostep/ostep11-threadintro/simple-race.s @@ -0,0 +1,6 @@ +.main +# this is a critical section +mov 2000(%bx), %ax # get the value at the address +add $1, %ax # increment it +mov %ax, 2000(%bx) # store it back +halt diff --git a/related_info/ostep/ostep11-threadintro/wait-for-me.s b/related_info/ostep/ostep11-threadintro/wait-for-me.s new file mode 100644 index 0000000..993606d --- /dev/null +++ b/related_info/ostep/ostep11-threadintro/wait-for-me.s @@ -0,0 +1,13 @@ +.main +test $1, %ax # ax should be 1 (signaller) or 0 (waiter) +je .signaller + +.waiter +mov 2000, %cx +test $1, %cx +jne .waiter +halt + +.signaller +mov $1, 2000 +halt diff --git a/related_info/ostep/ostep11-threadintro/x86.py b/related_info/ostep/ostep11-threadintro/x86.py new file mode 100755 index 0000000..6f10f93 --- /dev/null +++ b/related_info/ostep/ostep11-threadintro/x86.py @@ -0,0 +1,989 @@ +#! /usr/bin/env python + +import sys +import time +import random +from optparse import OptionParser + +# +# HELPER +# +def dospace(howmuch): + for i in range(howmuch): + print '%24s' % ' ', + +# useful instead of assert +def zassert(cond, str): + if cond == False: + print 'ABORT::', str + exit(1) + return + +class cpu: + # + # INIT: how much memory? + # + def __init__(self, memory, memtrace, regtrace, cctrace, compute, verbose): + # + # CONSTANTS + # + + # conditions + self.COND_GT = 0 + self.COND_GTE = 1 + self.COND_LT = 2 + self.COND_LTE = 3 + self.COND_EQ = 4 + self.COND_NEQ = 5 + + # registers in system + self.REG_ZERO = 0 + self.REG_AX = 1 + self.REG_BX = 2 + self.REG_CX = 3 + self.REG_DX = 4 + self.REG_SP = 5 + self.REG_BP = 6 + + # system memory: in KB + self.max_memory = memory * 1024 + + # which memory addrs and registers to trace? + self.memtrace = memtrace + self.regtrace = regtrace + self.cctrace = cctrace + self.compute = compute + self.verbose = verbose + + self.PC = 0 + self.registers = {} + self.conditions = {} + self.labels = {} + self.vars = {} + self.memory = {} + self.pmemory = {} # for printable version of what's in memory (instructions) + + self.condlist = [self.COND_GTE, self.COND_GT, self.COND_LTE, self.COND_LT, self.COND_NEQ, self.COND_EQ] + self.regnums = [self.REG_ZERO, self.REG_AX, self.REG_BX, self.REG_CX, self.REG_DX, self.REG_SP, self.REG_BP] + + self.regnames = {} + self.regnames['zero'] = self.REG_ZERO # hidden zero-valued register + self.regnames['ax'] = self.REG_AX + self.regnames['bx'] = self.REG_BX + self.regnames['cx'] = self.REG_CX + self.regnames['dx'] = self.REG_DX + self.regnames['sp'] = self.REG_SP + self.regnames['bp'] = self.REG_BP + + tmplist = [] + for r in self.regtrace: + zassert(r in self.regnames, 'Register %s cannot be traced because it does not exist' % r) + tmplist.append(self.regnames[r]) + self.regtrace = tmplist + + self.init_memory() + self.init_registers() + self.init_condition_codes() + + # + # BEFORE MACHINE RUNS + # + def init_condition_codes(self): + for c in self.condlist: + self.conditions[c] = False + + def init_memory(self): + for i in range(self.max_memory): + self.memory[i] = 0 + + def init_registers(self): + for i in self.regnums: + self.registers[i] = 0 + + def dump_memory(self): + print 'MEMORY DUMP' + for i in range(self.max_memory): + if i not in self.pmemory and i in self.memory and self.memory[i] != 0: + print ' m[%d]' % i, self.memory[i] + + # + # INFORMING ABOUT THE HARDWARE + # + def get_regnum(self, name): + assert(name in self.regnames) + return self.regnames[name] + + def get_regname(self, num): + assert(num in self.regnums) + for rname in self.regnames: + if self.regnames[rname] == num: + return rname + return '' + + def get_regnums(self): + return self.regnums + + def get_condlist(self): + return self.condlist + + def get_reg(self, reg): + assert(reg in self.regnums) + return self.registers[reg] + + def get_cond(self, cond): + assert(cond in self.condlist) + return self.conditions[cond] + + def get_pc(self): + return self.PC + + def set_reg(self, reg, value): + assert(reg in self.regnums) + self.registers[reg] = value + + def set_cond(self, cond, value): + assert(cond in self.condlist) + self.conditions[cond] = value + + def set_pc(self, pc): + self.PC = pc + + # + # INSTRUCTIONS + # + def halt(self): + return -1 + + def iyield(self): + return -2 + + def nop(self): + return 0 + + def rdump(self): + print 'REGISTERS::', + print 'ax:', self.registers[self.REG_AX], + print 'bx:', self.registers[self.REG_BX], + print 'cx:', self.registers[self.REG_CX], + print 'dx:', self.registers[self.REG_DX], + + def mdump(self, index): + print 'm[%d] ' % index, self.memory[index] + + def move_i_to_r(self, src, dst): + self.registers[dst] = src + return 0 + + # memory: value, register, register + def move_i_to_m(self, src, value, reg1, reg2): + tmp = value + self.registers[reg1] + self.registers[reg2] + self.memory[tmp] = src + return 0 + + def move_m_to_r(self, value, reg1, reg2, dst): + tmp = value + self.registers[reg1] + self.registers[reg2] + # print 'doing mov', 'val:', value, 'r1:', self.get_regname(reg1), self.registers[reg1], 'r2:', self.get_regname(reg2), self.registers[reg2], 'dst', self.get_regname(dst), 'tmp', tmp, 'reg[dst]', self.registers[dst], 'mem', self.memory[tmp] + self.registers[dst] = self.memory[tmp] + + def move_r_to_m(self, src, value, reg1, reg2): + tmp = value + self.registers[reg1] + self.registers[reg2] + self.memory[tmp] = self.registers[src] + return 0 + + def move_r_to_r(self, src, dst): + self.registers[dst] = self.registers[src] + return 0 + + def add_i_to_r(self, src, dst): + self.registers[dst] += src + return 0 + + def add_r_to_r(self, src, dst): + self.registers[dst] += self.registers[src] + return 0 + + def sub_i_to_r(self, src, dst): + self.registers[dst] -= src + return 0 + + def sub_r_to_r(self, src, dst): + self.registers[dst] -= self.registers[src] + return 0 + + + # + # SUPPORT FOR LOCKS + # + def atomic_exchange(self, src, value, reg1, reg2): + tmp = value + self.registers[reg1] + self.registers[reg2] + old = self.memory[tmp] + self.memory[tmp] = self.registers[src] + self.registers[src] = old + return 0 + + def fetchadd(self, src, value, reg1, reg2): + tmp = value + self.registers[reg1] + self.registers[reg2] + old = self.memory[tmp] + self.memory[tmp] = self.memory[tmp] + self.registers[src] + self.registers[src] = old + + # + # TEST for conditions + # + def test_all(self, src, dst): + self.init_condition_codes() + if dst > src: + self.conditions[self.COND_GT] = True + if dst >= src: + self.conditions[self.COND_GTE] = True + if dst < src: + self.conditions[self.COND_LT] = True + if dst <= src: + self.conditions[self.COND_LTE] = True + if dst == src: + self.conditions[self.COND_EQ] = True + if dst != src: + self.conditions[self.COND_NEQ] = True + return 0 + + def test_i_r(self, src, dst): + self.init_condition_codes() + return self.test_all(src, self.registers[dst]) + + def test_r_i(self, src, dst): + self.init_condition_codes() + return self.test_all(self.registers[src], dst) + + def test_r_r(self, src, dst): + self.init_condition_codes() + return self.test_all(self.registers[src], self.registers[dst]) + + # + # JUMPS + # + def jump(self, targ): + self.PC = targ + return 0 + + def jump_notequal(self, targ): + if self.conditions[self.COND_NEQ] == True: + self.PC = targ + return 0 + + def jump_equal(self, targ): + if self.conditions[self.COND_EQ] == True: + self.PC = targ + return 0 + + def jump_lessthan(self, targ): + if self.conditions[self.COND_LT] == True: + self.PC = targ + return 0 + + def jump_lessthanorequal(self, targ): + if self.conditions[self.COND_LTE] == True: + self.PC = targ + return 0 + + def jump_greaterthan(self, targ): + if self.conditions[self.COND_GT] == True: + self.PC = targ + return 0 + + def jump_greaterthanorequal(self, targ): + if self.conditions[self.COND_GTE] == True: + self.PC = targ + return 0 + + # + # CALL and RETURN + # + def call(self, targ): + self.registers[self.REG_SP] -= 4 + self.memory[self.registers[self.REG_SP]] = self.PC + self.PC = targ + + def ret(self): + self.PC = self.memory[self.registers[self.REG_SP]] + self.registers[self.REG_SP] += 4 + + # + # STACK and related + # + def push_r(self, reg): + self.registers[self.REG_SP] -= 4 + self.memory[self.registers[self.REG_SP]] = self.registers[reg] + return 0 + + def push_m(self, value, reg1, reg2): + # print 'push_m', value, reg1, reg2 + self.registers[self.REG_SP] -= 4 + tmp = value + self.registers[reg1] + self.registers[reg2] + # push address onto stack, not memory value itself + self.memory[self.registers[self.REG_SP]] = tmp + return 0 + + def pop(self): + self.registers[self.REG_SP] += 4 + + def pop_r(self, dst): + self.registers[dst] = self.registers[self.REG_SP] + self.registers[self.REG_SP] += 4 + + # + # HELPER func for getarg + # + def register_translate(self, r): + if r in self.regnames: + return self.regnames[r] + zassert(False, 'Register %s is not a valid register' % r) + return + + # + # HELPER in parsing mov (quite primitive) and other ops + # returns: (value, type) + # where type is (TYPE_REGISTER, TYPE_IMMEDIATE, TYPE_MEMORY) + # + # FORMATS + # %ax - register + # $10 - immediate + # 10 - direct memory + # 10(%ax) - memory + reg indirect + # 10(%ax,%bx) - memory + 2 reg indirect + # 10(%ax,%bx,4) - XXX (not handled) + # + def getarg(self, arg): + tmp1 = arg.replace(',', '') + tmp = tmp1.replace(' \t', '') + + if tmp[0] == '$': + zassert(len(tmp) == 2, 'correct form is $number (not %s)' % tmp) + value = tmp.split('$')[1] + zassert(value.isdigit(), 'value [%s] must be a digit' % value) + return int(value), 'TYPE_IMMEDIATE' + elif tmp[0] == '%': + register = tmp.split('%')[1] + return self.register_translate(register), 'TYPE_REGISTER' + elif tmp[0] == '(': + register = tmp.split('(')[1].split(')')[0].split('%')[1] + return '%d,%d,%d' % (0, self.register_translate(register), self.register_translate('zero')), 'TYPE_MEMORY' + elif tmp[0] == '.': + targ = tmp + return targ, 'TYPE_LABEL' + elif tmp[0].isalpha() and not tmp[0].isdigit(): + zassert(tmp in self.vars, 'Variable %s is not declared' % tmp) + # print '%d,%d,%d' % (self.vars[tmp], self.register_translate('zero'), self.register_translate('zero')), 'TYPE_MEMORY' + return '%d,%d,%d' % (self.vars[tmp], self.register_translate('zero'), self.register_translate('zero')), 'TYPE_MEMORY' + elif tmp[0].isdigit() or tmp[0] == '-': + # MOST GENERAL CASE: number(reg,reg) or number(reg) + # we ignore the common x86 number(reg,reg,constant) for now + neg = 1 + if tmp[0] == '-': + tmp = tmp[1:] + neg = -1 + s = tmp.split('(') + if len(s) == 1: + value = neg * int(tmp) + # print '%d,%d,%d' % (int(value), self.register_translate('zero'), self.register_translate('zero')), 'TYPE_MEMORY' + return '%d,%d,%d' % (int(value), self.register_translate('zero'), self.register_translate('zero')), 'TYPE_MEMORY' + elif len(s) == 2: + value = neg * int(s[0]) + t = s[1].split(')')[0].split(',') + if len(t) == 1: + register = t[0].split('%')[1] + # print '%d,%d,%d' % (int(value), self.register_translate(register), self.register_translate('zero')), 'TYPE_MEMORY' + return '%d,%d,%d' % (int(value), self.register_translate(register), self.register_translate('zero')), 'TYPE_MEMORY' + elif len(t) == 2: + register1 = t[0].split('%')[1] + register2 = t[1].split('%')[1] + # print '%d,%d,%d' % (int(value), self.register_translate(register1), self.register_translate(register2)), 'TYPE_MEMORY' + return '%d,%d,%d' % (int(value), self.register_translate(register1), self.register_translate(register2)), 'TYPE_MEMORY' + else: + print 'mov: bad argument [%s]' % tmp + exit(1) + return + zassert(True, 'mov: bad argument [%s]' % arg) + return + + # + # LOAD a program into memory + # make it ready to execute + # + def load(self, infile, loadaddr): + pc = int(loadaddr) + fd = open(infile) + + bpc = loadaddr + data = 100 + + for line in fd: + cline = line.rstrip() + # print 'PASS 1', cline + + # remove everything after the comment marker + ctmp = cline.split('#') + assert(len(ctmp) == 1 or len(ctmp) == 2) + if len(ctmp) == 2: + cline = ctmp[0] + + # remove empty lines, and split line by spaces + tmp = cline.split() + if len(tmp) == 0: + continue + + # only pay attention to labels and variables + if tmp[0] == '.var': + assert(len(tmp) == 2) + assert(tmp[0] not in self.vars) + self.vars[tmp[1]] = data + data += 4 + zassert(data < bpc, 'Load address overrun by static data') + if self.verbose: print 'ASSIGN VAR', tmp[0], "-->", tmp[1], self.vars[tmp[1]] + elif tmp[0][0] == '.': + assert(len(tmp) == 1) + self.labels[tmp[0]] = int(pc) + if self.verbose: print 'ASSIGN LABEL', tmp[0], "-->", pc + else: + pc += 1 + fd.close() + + if self.verbose: print '' + + # second pass: do everything else + pc = int(loadaddr) + fd = open(infile) + for line in fd: + cline = line.rstrip() + # print 'PASS 2', cline + + # remove everything after the comment marker + ctmp = cline.split('#') + assert(len(ctmp) == 1 or len(ctmp) == 2) + if len(ctmp) == 2: + cline = ctmp[0] + + # remove empty lines, and split line by spaces + tmp = cline.split() + if len(tmp) == 0: + continue + + # skip labels: all else must be instructions + if cline[0] != '.': + tmp = cline.split(None, 1) + opcode = tmp[0] + self.pmemory[pc] = cline.strip() + + # MAIN OPCODE LOOP + if opcode == 'mov': + rtmp = tmp[1].split(',', 1) + zassert(len(tmp) == 2 and len(rtmp) == 2, 'mov: needs two args, separated by commas [%s]' % cline) + arg1 = rtmp[0].strip() + arg2 = rtmp[1].strip() + (src, stype) = self.getarg(arg1) + (dst, dtype) = self.getarg(arg2) + # print 'MOV', src, stype, dst, dtype + if stype == 'TYPE_MEMORY' and dtype == 'TYPE_MEMORY': + print 'bad mov: two memory arguments' + exit(1) + elif stype == 'TYPE_IMMEDIATE' and dtype == 'TYPE_IMMEDIATE': + print 'bad mov: two immediate arguments' + exit(1) + elif stype == 'TYPE_IMMEDIATE' and dtype == 'TYPE_REGISTER': + self.memory[pc] = 'self.move_i_to_r(%d, %d)' % (int(src), dst) + elif stype == 'TYPE_IMMEDIATE' and dtype == 'TYPE_REGISTER': + self.memory[pc] = 'self.move_i_to_r(%d, %d)' % (int(src), dst) + elif stype == 'TYPE_MEMORY' and dtype == 'TYPE_REGISTER': + tmp = src.split(',') + assert(len(tmp) == 3) + self.memory[pc] = 'self.move_m_to_r(%d, %d, %d, %d)' % (int(tmp[0]), int(tmp[1]), int(tmp[2]), dst) + elif stype == 'TYPE_REGISTER' and dtype == 'TYPE_MEMORY': + tmp = dst.split(',') + assert(len(tmp) == 3) + self.memory[pc] = 'self.move_r_to_m(%d, %d, %d, %d)' % (src, int(tmp[0]), int(tmp[1]), int(tmp[2])) + elif stype == 'TYPE_REGISTER' and dtype == 'TYPE_REGISTER': + self.memory[pc] = 'self.move_r_to_r(%d, %d)' % (src, dst) + elif stype == 'TYPE_IMMEDIATE' and dtype == 'TYPE_MEMORY': + tmp = dst.split(',') + assert(len(tmp) == 3) + self.memory[pc] = 'self.move_i_to_m(%d, %d, %d, %d)' % (src, int(tmp[0]), int(tmp[1]), int(tmp[2])) + else: + zassert(False, 'malformed mov instruction') + elif opcode == 'pop': + if len(tmp) == 1: + self.memory[pc] = 'self.pop()' + elif len(tmp) == 2: + arg = tmp[1].strip() + (dst, dtype) = self.getarg(arg) + zassert(dtype == 'TYPE_REGISTER', 'Can only pop into a register') + self.memory[pc] = 'self.pop_r(%d)' % dst + else: + zassert(False, 'pop instruction must take zero/one args') + elif opcode == 'push': + (src, stype) = self.getarg(tmp[1].strip()) + if stype == 'TYPE_REGISTER': + self.memory[pc] = 'self.push_r(%d)' % (int(src)) + elif stype == 'TYPE_MEMORY': + tmp = src.split(',') + assert(len(tmp) == 3) + self.memory[pc] = 'self.push_m(%d,%d,%d)' % (int(tmp[0]), int(tmp[1]), int(tmp[2])) + else: + zassert(False, 'Cannot push anything but registers') + elif opcode == 'call': + (targ, ttype) = self.getarg(tmp[1].strip()) + if ttype == 'TYPE_LABEL': + self.memory[pc] = 'self.call(%d)' % (int(self.labels[targ])) + else: + zassert(False, 'Cannot call anything but a label') + elif opcode == 'ret': + assert(len(tmp) == 1) + self.memory[pc] = 'self.ret()' + elif opcode == 'add': + rtmp = tmp[1].split(',', 1) + zassert(len(tmp) == 2 and len(rtmp) == 2, 'add: needs two args, separated by commas [%s]' % cline) + arg1 = rtmp[0].strip() + arg2 = rtmp[1].strip() + (src, stype) = self.getarg(arg1) + (dst, dtype) = self.getarg(arg2) + if stype == 'TYPE_IMMEDIATE' and dtype == 'TYPE_REGISTER': + self.memory[pc] = 'self.add_i_to_r(%d, %d)' % (int(src), dst) + elif stype == 'TYPE_REGISTER' and dtype == 'TYPE_REGISTER': + self.memory[pc] = 'self.add_r_to_r(%d, %d)' % (int(src), dst) + else: + zassert(False, 'malformed usage of add instruction') + elif opcode == 'sub': + rtmp = tmp[1].split(',', 1) + zassert(len(tmp) == 2 and len(rtmp) == 2, 'sub: needs two args, separated by commas [%s]' % cline) + arg1 = rtmp[0].strip() + arg2 = rtmp[1].strip() + (src, stype) = self.getarg(arg1) + (dst, dtype) = self.getarg(arg2) + if stype == 'TYPE_IMMEDIATE' and dtype == 'TYPE_REGISTER': + self.memory[pc] = 'self.sub_i_to_r(%d, %d)' % (int(src), dst) + elif stype == 'TYPE_REGISTER' and dtype == 'TYPE_REGISTER': + self.memory[pc] = 'self.sub_r_to_r(%d, %d)' % (int(src), dst) + else: + zassert(False, 'malformed usage of sub instruction') + elif opcode == 'fetchadd': + rtmp = tmp[1].split(',', 1) + zassert(len(tmp) == 2 and len(rtmp) == 2, 'fetchadd: needs two args, separated by commas [%s]' % cline) + arg1 = rtmp[0].strip() + arg2 = rtmp[1].strip() + (src, stype) = self.getarg(arg1) + (dst, dtype) = self.getarg(arg2) + tmp = dst.split(',') + assert(len(tmp) == 3) + if stype == 'TYPE_REGISTER' and dtype == 'TYPE_MEMORY': + self.memory[pc] = 'self.fetchadd(%d, %d, %d, %d)' % (src, int(tmp[0]), int(tmp[1]), int(tmp[2])) + else: + zassert(False, 'poorly specified fetch and add') + elif opcode == 'xchg': + rtmp = tmp[1].split(',', 1) + zassert(len(tmp) == 2 and len(rtmp) == 2, 'xchg: needs two args, separated by commas [%s]' % cline) + arg1 = rtmp[0].strip() + arg2 = rtmp[1].strip() + (src, stype) = self.getarg(arg1) + (dst, dtype) = self.getarg(arg2) + tmp = dst.split(',') + assert(len(tmp) == 3) + if stype == 'TYPE_REGISTER' and dtype == 'TYPE_MEMORY': + self.memory[pc] = 'self.atomic_exchange(%d, %d, %d, %d)' % (src, int(tmp[0]), int(tmp[1]), int(tmp[2])) + else: + zassert(False, 'poorly specified atomic exchange') + elif opcode == 'test': + rtmp = tmp[1].split(',', 1) + zassert(len(tmp) == 2 and len(rtmp) == 2, 'test: needs two args, separated by commas [%s]' % cline) + arg1 = rtmp[0].strip() + arg2 = rtmp[1].strip() + (src, stype) = self.getarg(arg1) + (dst, dtype) = self.getarg(arg2) + if stype == 'TYPE_IMMEDIATE' and dtype == 'TYPE_REGISTER': + self.memory[pc] = 'self.test_i_r(%d, %d)' % (int(src), dst) + elif stype == 'TYPE_REGISTER' and dtype == 'TYPE_REGISTER': + self.memory[pc] = 'self.test_r_r(%d, %d)' % (int(src), dst) + elif stype == 'TYPE_REGISTER' and dtype == 'TYPE_IMMEDIATE': + self.memory[pc] = 'self.test_r_i(%d, %d)' % (int(src), dst) + else: + zassert(False, 'malformed usage of test instruction') + elif opcode == 'j': + (targ, ttype) = self.getarg(tmp[1].strip()) + zassert(ttype == 'TYPE_LABEL', 'bad jump target [%s]' % tmp[1].strip()) + self.memory[pc] = 'self.jump(%d)' % int(self.labels[targ]) + elif opcode == 'jne': + (targ, ttype) = self.getarg(tmp[1].strip()) + zassert(ttype == 'TYPE_LABEL', 'bad jump target [%s]' % tmp[1].strip()) + self.memory[pc] = 'self.jump_notequal(%d)' % int(self.labels[targ]) + elif opcode == 'je': + (targ, ttype) = self.getarg(tmp[1].strip()) + zassert(ttype == 'TYPE_LABEL', 'bad jump target [%s]' % tmp[1].strip()) + self.memory[pc] = 'self.jump_equal(%d)' % self.labels[targ] + elif opcode == 'jlt': + (targ, ttype) = self.getarg(tmp[1].strip()) + zassert(ttype == 'TYPE_LABEL', 'bad jump target [%s]' % tmp[1].strip()) + self.memory[pc] = 'self.jump_lessthan(%d)' % int(self.labels[targ]) + elif opcode == 'jlte': + (targ, ttype) = self.getarg(tmp[1].strip()) + zassert(ttype == 'TYPE_LABEL', 'bad jump target [%s]' % tmp[1].strip()) + self.memory[pc] = 'self.jump_lessthanorequal(%s)' % self.labels[targ] + elif opcode == 'jgt': + (targ, ttype) = self.getarg(tmp[1].strip()) + zassert(ttype == 'TYPE_LABEL', 'bad jump target [%s]' % tmp[1].strip()) + self.memory[pc] = 'self.jump_greaterthan(%d)' % int(self.labels[targ]) + elif opcode == 'jgte': + (targ, ttype) = self.getarg(tmp[1].strip()) + zassert(ttype == 'TYPE_LABEL', 'bad jump target [%s]' % tmp[1].strip()) + self.memory[pc] = 'self.jump_greaterthanorequal(%s)' % self.labels[targ] + elif opcode == 'nop': + self.memory[pc] = 'self.nop()' + elif opcode == 'halt': + self.memory[pc] = 'self.halt()' + elif opcode == 'yield': + self.memory[pc] = 'self.iyield()' + elif opcode == 'rdump': + self.memory[pc] = 'self.rdump()' + elif opcode == 'mdump': + self.memory[pc] = 'self.mdump(%s)' % tmp[1] + else: + print 'illegal opcode: ', opcode + exit(1) + + if self.verbose: print 'pc:%d LOADING %20s --> %s' % (pc, self.pmemory[pc], self.memory[pc]) + + # INCREMENT PC for loader + pc += 1 + # END: loop over file + fd.close() + if self.verbose: print '' + return + # END: load + + def print_headers(self, procs): + # print some headers + if len(self.memtrace) > 0: + for m in self.memtrace: + if m[0].isdigit(): + print '%5d' % int(m), + else: + zassert(m in self.vars, 'Traced variable %s not declared' % m) + print '%5s' % m, + print ' ', + if len(self.regtrace) > 0: + for r in self.regtrace: + print '%5s' % self.get_regname(r), + print ' ', + if cctrace == True: + print '>= > <= < != ==', + + # and per thread + for i in range(procs.getnum()): + print ' Thread %d ' % i, + print '' + return + + def print_trace(self, newline): + if len(self.memtrace) > 0: + for m in self.memtrace: + if self.compute: + if m[0].isdigit(): + print '%5d' % self.memory[int(m)], + else: + zassert(m in self.vars, 'Traced variable %s not declared' % m) + print '%5d' % self.memory[self.vars[m]], + else: + print '%5s' % '?', + print ' ', + if len(self.regtrace) > 0: + for r in self.regtrace: + if self.compute: + print '%5d' % self.registers[r], + else: + print '%5s' % '?', + print ' ', + if cctrace == True: + for c in self.condlist: + if self.compute: + if self.conditions[c]: + print '1 ', + else: + print '0 ', + else: + print '? ', + if (len(self.memtrace) > 0 or len(self.regtrace) > 0 or cctrace == True) and newline == True: + print '' + return + + def setint(self, intfreq, intrand): + if intrand == False: + return intfreq + return int(random.random() * intfreq) + 1 + + def run(self, procs, intfreq, intrand): + # hw init: cc's, interrupt frequency, etc. + interrupt = self.setint(intfreq, intrand) + icount = 0 + + self.print_headers(procs) + self.print_trace(True) + + while True: + # need thread ID of current process + tid = procs.getcurr().gettid() + + # FETCH + prevPC = self.PC + instruction = self.memory[self.PC] + self.PC += 1 + + # DECODE and EXECUTE + # key: self.PC may be changed during eval; thus MUST be incremented BEFORE eval + rc = eval(instruction) + + # tracing details: ALWAYS AFTER EXECUTION OF INSTRUCTION + self.print_trace(False) + + # output: thread-proportional spacing followed by PC and instruction + dospace(tid) + print prevPC, self.pmemory[prevPC] + icount += 1 + + # halt instruction issued + if rc == -1: + procs.done() + if procs.numdone() == procs.getnum(): + return icount + procs.next() + procs.restore() + + self.print_trace(False) + for i in range(procs.getnum()): + print '----- Halt;Switch ----- ', + print '' + + # do interrupt processing + interrupt -= 1 + if interrupt == 0 or rc == -2: + interrupt = self.setint(intfreq, intrand) + procs.save() + procs.next() + procs.restore() + + self.print_trace(False) + for i in range(procs.getnum()): + print '------ Interrupt ------ ', + print '' + # END: while + return + +# +# END: class cpu +# + + +# +# PROCESS LIST class +# +class proclist: + def __init__(self): + self.plist = [] + self.curr = 0 + self.active = 0 + + def done(self): + self.plist[self.curr].setdone() + self.active -= 1 + + def numdone(self): + return len(self.plist) - self.active + + def getnum(self): + return len(self.plist) + + def add(self, p): + self.active += 1 + self.plist.append(p) + + def getcurr(self): + return self.plist[self.curr] + + def save(self): + self.plist[self.curr].save() + + def restore(self): + self.plist[self.curr].restore() + + def next(self): + for i in range(self.curr+1, len(self.plist)): + if self.plist[i].isdone() == False: + self.curr = i + return + for i in range(0, self.curr+1): + if self.plist[i].isdone() == False: + self.curr = i + return + +# +# PROCESS class +# +class process: + def __init__(self, cpu, tid, pc, stackbottom, reginit): + self.cpu = cpu # object reference + self.tid = tid + self.pc = pc + self.regs = {} + self.cc = {} + self.done = False + self.stack = stackbottom + + # init regs: all 0 or specially set to something + for r in self.cpu.get_regnums(): + self.regs[r] = 0 + if reginit != '': + # form: ax=1,bx=2 (for some subset of registers) + for r in reginit.split(':'): + tmp = r.split('=') + assert(len(tmp) == 2) + self.regs[self.cpu.get_regnum(tmp[0])] = int(tmp[1]) + + # init CCs + for c in self.cpu.get_condlist(): + self.cc[c] = False + + # stack + self.regs[self.cpu.get_regnum('sp')] = stackbottom + # print 'REG', self.cpu.get_regnum('sp'), self.regs[self.cpu.get_regnum('sp')] + + return + + def gettid(self): + return self.tid + + def save(self): + self.pc = self.cpu.get_pc() + for c in self.cpu.get_condlist(): + self.cc[c] = self.cpu.get_cond(c) + for r in self.cpu.get_regnums(): + self.regs[r] = self.cpu.get_reg(r) + + def restore(self): + self.cpu.set_pc(self.pc) + for c in self.cpu.get_condlist(): + self.cpu.set_cond(c, self.cc[c]) + for r in self.cpu.get_regnums(): + self.cpu.set_reg(r, self.regs[r]) + + def setdone(self): + self.done = True + + def isdone(self): + return self.done == True + +# +# main program +# +parser = OptionParser() +parser.add_option('-s', '--seed', default=0, help='the random seed', action='store', type='int', dest='seed') +parser.add_option('-t', '--threads', default=2, help='number of threads', action='store', type='int', dest='numthreads') +parser.add_option('-p', '--program', default='', help='source program (in .s)', action='store', type='string', dest='progfile') +parser.add_option('-i', '--interrupt', default=50, help='interrupt frequency', action='store', type='int', dest='intfreq') +parser.add_option('-r', '--randints', default=False, help='if interrupts are random', action='store_true', dest='intrand') +parser.add_option('-a', '--argv', default='', + help='comma-separated per-thread args (e.g., ax=1,ax=2 sets thread 0 ax reg to 1 and thread 1 ax reg to 2); specify multiple regs per thread via colon-separated list (e.g., ax=1:bx=2,cx=3 sets thread 0 ax and bx and just cx for thread 1)', + action='store', type='string', dest='argv') +parser.add_option('-L', '--loadaddr', default=1000, help='address where to load code', action='store', type='int', dest='loadaddr') +parser.add_option('-m', '--memsize', default=128, help='size of address space (KB)', action='store', type='int', dest='memsize') +parser.add_option('-M', '--memtrace', default='', help='comma-separated list of addrs to trace (e.g., 20000,20001)', action='store', + type='string', dest='memtrace') +parser.add_option('-R', '--regtrace', default='', help='comma-separated list of regs to trace (e.g., ax,bx,cx,dx)', action='store', + type='string', dest='regtrace') +parser.add_option('-C', '--cctrace', default=False, help='should we trace condition codes', action='store_true', dest='cctrace') +parser.add_option('-S', '--printstats',default=False, help='print some extra stats', action='store_true', dest='printstats') +parser.add_option('-v', '--verbose', default=False, help='print some extra info', action='store_true', dest='verbose') +parser.add_option('-c', '--compute', default=False, help='compute answers for me', action='store_true', dest='solve') +(options, args) = parser.parse_args() + +print 'ARG seed', options.seed +print 'ARG numthreads', options.numthreads +print 'ARG program', options.progfile +print 'ARG interrupt frequency', options.intfreq +print 'ARG interrupt randomness',options.intrand +print 'ARG argv', options.argv +print 'ARG load address', options.loadaddr +print 'ARG memsize', options.memsize +print 'ARG memtrace', options.memtrace +print 'ARG regtrace', options.regtrace +print 'ARG cctrace', options.cctrace +print 'ARG printstats', options.printstats +print 'ARG verbose', options.verbose +print '' + +seed = int(options.seed) +numthreads = int(options.numthreads) +intfreq = int(options.intfreq) +zassert(intfreq > 0, 'Interrupt frequency must be greater than 0') +intrand = int(options.intrand) +progfile = options.progfile +zassert(progfile != '', 'Program file must be specified') +argv = options.argv.split(',') +zassert(len(argv) == numthreads or len(argv) == 1, 'argv: must be one per-thread or just one set of values for all threads') + +loadaddr = options.loadaddr +memsize = options.memsize + +memtrace = [] +if options.memtrace != '': + for m in options.memtrace.split(','): + memtrace.append(m) + +regtrace = [] +if options.regtrace != '': + for r in options.regtrace.split(','): + regtrace.append(r) + +cctrace = options.cctrace + +printstats = options.printstats +verbose = options.verbose + +# +# MAIN program +# +debug = False +debug = False + +cpu = cpu(memsize, memtrace, regtrace, cctrace, options.solve, verbose) + +# load a program +cpu.load(progfile, loadaddr) + +# process list +procs = proclist() +pid = 0 +stack = memsize * 1000 +for t in range(numthreads): + if len(argv) > 1: + arg = argv[pid] + else: + arg = argv[0] + procs.add(process(cpu, pid, loadaddr, stack, arg)) + stack -= 1000 + pid += 1 + +# get first one ready! +procs.restore() + +# run it +t1 = time.clock() +ic = cpu.run(procs, intfreq, intrand) +t2 = time.clock() + +if printstats: + print '' + print 'STATS:: Instructions %d' % ic + print 'STATS:: Emulation Rate %.2f kinst/sec' % (float(ic) / float(t2 - t1) / 1000.0) + +# use this for profiling +# import cProfile +# cProfile.run('run()') + + + + diff --git a/related_info/ostep/ostep12-threadlock/flag.s b/related_info/ostep/ostep12-threadlock/flag.s new file mode 100644 index 0000000..8a3f2f0 --- /dev/null +++ b/related_info/ostep/ostep12-threadlock/flag.s @@ -0,0 +1,27 @@ +.var flag +.var count + +.main +.top + +.acquire +mov flag, %ax # get flag +test $0, %ax # if we get 0 back: lock is free! +jne .acquire # if not, try again +mov $1, flag # store 1 into flag + +# critical section +mov count, %ax # get the value at the address +add $1, %ax # increment it +mov %ax, count # store it back + +# release lock +mov $0, flag # clear the flag now + +# see if we're still looping +sub $1, %bx +test $0, %bx +jgt .top + +halt + diff --git a/related_info/ostep/ostep12-threadlock/locks.md b/related_info/ostep/ostep12-threadlock/locks.md new file mode 100644 index 0000000..2d8353c --- /dev/null +++ b/related_info/ostep/ostep12-threadlock/locks.md @@ -0,0 +1,351 @@ + +Welcome to this simulator. The idea is to gain familiarity with threads by +seeing how they interleave; the simulator, x86.py, will help you in +gaining this understanding. + +The simulator mimicks the execution of short assembly sequences by multiple +threads. Note that the OS code that would run (for example, to perform a +context switch) is *not* shown; thus, all you see is the interleaving of the +user code. + +The assembly code that is run is based on x86, but somewhat simplified. +In this instruction set, there are four general-purpose registers +(%ax, %bx, %cx, %dx), a program counter (PC), and a small set of instructions +which will be enough for our purposes. We've also added a few extra GP +registers (%ex, %fx) which don't quite match anything in x86 land +(but that is OK). + +Here is an example code snippet that we will be able to run: + +.main +mov 2000, %ax # get the value at the address +add $1, %ax # increment it +mov %ax, 2000 # store it back +halt + +The code is easy to understand. The first instruction, an x86 "mov", simply +loads a value from the address specified by 2000 into the register %ax. +Addresses, in this subset of x86, can take some of the following forms: + + 2000 -> the number (2000) is the address + (%cx) -> contents of register (in parentheses) forms the address + 1000(%dx) -> the number + contents of the register form the address + 10(%ax,%bx) -> the number + reg1 + reg2 forms the address + 10(%ax,%bx,4) -> the number + reg1 + (reg2*scaling) forms the address + +To store a value, the same "mov" instruction is used, but this time with the +arguments reversed, e.g.: + + mov %ax, 2000 + +The "add" instruction, from the sequence above, should be clear: it adds an +immediate value (specified by $1) to the register specified in the second +argument (i.e., %ax = %ax + 1). + +Thus, we now can understand the code sequence above: it loads the value at +address 2000, adds 1 to it, and then stores the value back into address 2000. + +The fake-ish "halt" instruction just stops running this thread. + +Let's run the simulator and see how this all works! Assume the above code +sequence is in the file "simple-race.s". + +prompt> ./x86.py -p simple-race.s -t 1 + + Thread 0 +1000 mov 2000, %ax +1001 add $1, %ax +1002 mov %ax, 2000 +1003 halt + +prompt> + +The arguments used here specify the program (-p), the number of threads (-t +1), and the interrupt interval, which is how often a scheduler will be woken +and run to switch to a different task. Because there is only one thread in +this example, this interval does not matter. + +The output is easy to read: the simulator prints the program counter (here +shown from 1000 to 1003) and the instruction that gets executed. Note that we +assume (unrealistically) that all instructions just take up a single byte in +memory; in x86, instructions are variable-sized and would take up from one to +a small number of bytes. + +We can use more detailed tracing to get a better sense of how machine state +changes during the execution: + +prompt> ./x86.py -p simple-race.s -t 1 -M 2000 -R ax,bx + + 2000 ax bx Thread 0 + ? ? ? + ? ? ? 1000 mov 2000, %ax + ? ? ? 1001 add $1, %ax + ? ? ? 1002 mov %ax, 2000 + ? ? ? 1003 halt + +Oops! Forgot the -c flag (which actually computes the answers for you). + +prompt> ./x86.py -p simple-race.s -t 1 -M 2000 -R ax,bx -c + + 2000 ax bx Thread 0 + 0 0 0 + 0 0 0 1000 mov 2000, %ax + 0 1 0 1001 add $1, %ax + 1 1 0 1002 mov %ax, 2000 + 1 1 0 1003 halt + +By using the -M flag, we can trace memory locations (a comma-separated list +lets you trace more than one, e.g., 2000,3000); by using the -R flag we can +track the values inside specific registers. + +The values on the left show the memory/register contents AFTER the instruction +on the right has executed. For example, after the "add" instruction, you can +see that %ax has been incremented to the value 1; after the second "mov" +instruction (at PC=1002), you can see that the memory contents at 2000 are +now also incremented. + +There are a few more instructions you'll need to know, so let's get to them +now. Here is a code snippet of a loop: + +.main +.top +sub $1,%dx +test $0,%dx +jgte .top +halt + +A few things have been introduced here. First is the "test" instruction. +This instruction takes two arguments and compares them; it then sets implicit +"condition codes" (kind of like 1-bit registers) which subsequent instructions +can act upon. + +In this case, the other new instruction is the "jump" instruction (in this +case, "jgte" which stands for "jump if greater than or equal to"). This +instruction jumps if the first value is greater than or equal to the second +in the test. + +One last point: to really make this code work, dx must be initialized to 1 or +greater. + +Thus, we run the program like this: + +prompt> ./x86.py -p loop.s -t 1 -a dx=3 -R dx -C -c + + dx >= > <= < != == Thread 0 + 3 0 0 0 0 0 0 + 2 0 0 0 0 0 0 1000 sub $1,%dx + 2 1 1 0 0 1 0 1001 test $0,%dx + 2 1 1 0 0 1 0 1002 jgte .top + 1 1 1 0 0 1 0 1000 sub $1,%dx + 1 1 1 0 0 1 0 1001 test $0,%dx + 1 1 1 0 0 1 0 1002 jgte .top + 0 1 1 0 0 1 0 1000 sub $1,%dx + 0 1 0 1 0 0 1 1001 test $0,%dx + 0 1 0 1 0 0 1 1002 jgte .top + 0 1 0 1 0 0 1 1003 halt + +The "-R dx" flag traces the value of %dx; the "-C" flag traces the values of +the condition codes that get set by a test instruction. Finally, the "-a dx=3" +flag sets the %dx register to the value 3 to start with. + +As you can see from the trace, the "sub" instruction slowly lowers the value +of %dx. The first few times "test" is called, only the ">=", ">", and "!=" +conditions get set. However, the last "test" in the trace finds %dx and 0 to +be equal, and thus the subsequent jump does NOT take place, and the program +finally halts. + +Now, finally, we get to a more interesting case, i.e., a race condition with +multiple threads. Let's look at the code first: + +.main +.top +# critical section +mov 2000, %ax # get the value at the address +add $1, %ax # increment it +mov %ax, 2000 # store it back + +# see if we're still looping +sub $1, %bx +test $0, %bx +jgt .top + +halt + +The code has a critical section which loads the value of a variable +(at address 2000), then adds 1 to the value, then stores it back. + +The code after just decrements a loop counter (in %bx), tests if it +is greater than or equal to zero, and if so, jumps back to the top +to the critical section again. + +prompt> ./x86.py -p looping-race-nolock.s -t 2 -a bx=1 -M 2000 -c + + 2000 bx Thread 0 Thread 1 + 0 1 + 0 1 1000 mov 2000, %ax + 0 1 1001 add $1, %ax + 1 1 1002 mov %ax, 2000 + 1 0 1003 sub $1, %bx + 1 0 1004 test $0, %bx + 1 0 1005 jgt .top + 1 0 1006 halt + 1 1 ----- Halt;Switch ----- ----- Halt;Switch ----- + 1 1 1000 mov 2000, %ax + 1 1 1001 add $1, %ax + 2 1 1002 mov %ax, 2000 + 2 0 1003 sub $1, %bx + 2 0 1004 test $0, %bx + 2 0 1005 jgt .top + 2 0 1006 halt + +Here you can see each thread ran once, and each updated the shared +variable at address 2000 once, thus resulting in a count of two there. + +The "Halt;Switch" line is inserted whenever a thread halts and another +thread must be run. + +One last example: run the same thing above, but with a smaller interrupt +frequency. Here is what that will look like: + +[mac Race-Analyze] ./x86.py -p looping-race-nolock.s -t 2 -a bx=1 -M 2000 -i 2 + + 2000 Thread 0 Thread 1 + ? + ? 1000 mov 2000, %ax + ? 1001 add $1, %ax + ? ------ Interrupt ------ ------ Interrupt ------ + ? 1000 mov 2000, %ax + ? 1001 add $1, %ax + ? ------ Interrupt ------ ------ Interrupt ------ + ? 1002 mov %ax, 2000 + ? 1003 sub $1, %bx + ? ------ Interrupt ------ ------ Interrupt ------ + ? 1002 mov %ax, 2000 + ? 1003 sub $1, %bx + ? ------ Interrupt ------ ------ Interrupt ------ + ? 1004 test $0, %bx + ? 1005 jgt .top + ? ------ Interrupt ------ ------ Interrupt ------ + ? 1004 test $0, %bx + ? 1005 jgt .top + ? ------ Interrupt ------ ------ Interrupt ------ + ? 1006 halt + ? ----- Halt;Switch ----- ----- Halt;Switch ----- + ? 1006 halt + +As you can see, each thread is interrupt every 2 instructions, as we specify +via the "-i 2" flag. What is the value of memory[2000] throughout this run? +What should it have been? + +Now let's give a little more information on what can be simulated +with this program. The full set of registers: %ax, %bx, %cx, %dx, and the PC. +In this version, there is no support for a "stack", nor are there call +and return instructions. + +The full set of instructions simulated are: + +mov immediate, register # moves immediate value to register +mov memory, register # loads from memory into register +mov register, register # moves value from one register to other +mov register, memory # stores register contents in memory +mov immediate, memory # stores immediate value in memory + +add immediate, register # register = register + immediate +add register1, register2 # register2 = register2 + register1 +sub immediate, register # register = register - immediate +sub register1, register2 # register2 = register2 - register1 + +neg register # negates contents of register + +test immediate, register # compare immediate and register (set condition codes) +test register, immediate # same but register and immediate +test register, register # same but register and register + +jne # jump if test'd values are not equal +je # ... equal +jlt # ... second is less than first +jlte # ... less than or equal +jgt # ... is greater than +jgte # ... greater than or equal + +push memory or register # push value in memory or from reg onto stack + # stack is defined by sp register +pop [register] # pop value off stack (into optional register) +call label # call function at label + +xchg register, memory # atomic exchange: + # put value of register into memory + # return old contents of memory into reg + # do both things atomically + +yield # switch to the next thread in the runqueue + +nop # no op + + +Notes: +- 'immediate' is something of the form $number +- 'memory' is of the form 'number' or '(reg)' or 'number(reg)' or + 'number(reg,reg)' or 'number(reg,reg,scale)' (as described above) +- 'register' is one of %ax, %bx, %cx, %dx + +Finally, here are the full set of options to the simulator are available with +the -h flag: + +Usage: x86.py [options] + +Options: + -s SEED, --seed=SEED the random seed + -t NUMTHREADS, --threads=NUMTHREADS + number of threads + -p PROGFILE, --program=PROGFILE + source program (in .s) + -i INTFREQ, --interrupt=INTFREQ + interrupt frequency + -P PROCSCHED, --procsched=PROCSCHED + control exactly which thread runs when + -r, --randints if interrupts are random + -a ARGV, --argv=ARGV comma-separated per-thread args (e.g., ax=1,ax=2 sets + thread 0 ax reg to 1 and thread 1 ax reg to 2); + specify multiple regs per thread via colon-separated + list (e.g., ax=1:bx=2,cx=3 sets thread 0 ax and bx and + just cx for thread 1) + -L LOADADDR, --loadaddr=LOADADDR + address where to load code + -m MEMSIZE, --memsize=MEMSIZE + size of address space (KB) + -M MEMTRACE, --memtrace=MEMTRACE + comma-separated list of addrs to trace (e.g., + 20000,20001) + -R REGTRACE, --regtrace=REGTRACE + comma-separated list of regs to trace (e.g., + ax,bx,cx,dx) + -C, --cctrace should we trace condition codes + -S, --printstats print some extra stats + -v, --verbose print some extra info + -H HEADERCOUNT, --headercount=HEADERCOUNT + how often to print a row header + -c, --compute compute answers for me + + +Most are obvious. Usage of -r turns on a random interrupter (from 1 to intfreq +as specified by -i), which can make for more fun during homework problems. + +-P lets you specify exactly which threads run when; + e.g., 11000 would run thread 1 for 2 instructions, then thread 0 for 3, + then repeat + +-L specifies where in the address space to load the code. + +-m specified the size of the address space (in KB). + +-S prints some extra stats + +-c lets you see the values of the traced registers or memory values + (otherwise they show up as question marks) + +-H lets you specify how often to print a row header (useful for long traces) + +Now you have the basics in place; read the questions at the end of the chapter +to study this race condition and related issues in more depth. + diff --git a/related_info/ostep/ostep12-threadlock/peterson.s b/related_info/ostep/ostep12-threadlock/peterson.s new file mode 100644 index 0000000..8754695 --- /dev/null +++ b/related_info/ostep/ostep12-threadlock/peterson.s @@ -0,0 +1,53 @@ +# array of 2 integers (each size 4 bytes) +# load address of flag into fx register +# access flag[] with 0(%fx,%index,4) +# where %index is a register holding 0 or 1 +# index reg contains 0 -> flag[0], if 1->flag[1] +.var flag 2 + +# global turn variable +.var turn + +# global count +.var count + +.main + +# put address of flag into fx +lea flag, %fx + +# assume thread ID is in bx (0 or 1, scale by 4 to get proper flag address) +mov %bx, %cx # bx: self, now copies to cx +neg %cx # cx: - self +add $1, %cx # cx: 1 - self + +.acquire +mov $1, 0(%fx,%bx,4) # flag[self] = 1 +mov %cx, turn # turn = 1 - self + +.spin1 +mov 0(%fx,%cx,4), %ax # flag[1-self] +test $1, %ax +jne .fini # if flag[1-self] != 1, skip past loop to .fini + +.spin2 # just labeled for fun, not needed +mov turn, %ax +test %cx, %ax # compare 'turn' and '1 - self' +je .spin1 # if turn==1-self, go back and start spin again + +# fall out of spin +.fini + +# do critical section now +mov count, %ax +add $1, %ax +mov %ax, count + +.release +mov $0, 0(%fx,%bx,4) # flag[self] = 0 + + +# end case: make sure it's other's turn +mov %cx, turn # turn = 1 - self +halt + diff --git a/related_info/ostep/ostep12-threadlock/test-and-set.s b/related_info/ostep/ostep12-threadlock/test-and-set.s new file mode 100644 index 0000000..ad2a901 --- /dev/null +++ b/related_info/ostep/ostep12-threadlock/test-and-set.s @@ -0,0 +1,26 @@ +.var mutex +.var count + +.main +.top + +.acquire +mov $1, %ax +xchg %ax, mutex # atomic swap of 1 and mutex +test $0, %ax # if we get 0 back: lock is free! +jne .acquire # if not, try again + +# critical section +mov count, %ax # get the value at the address +add $1, %ax # increment it +mov %ax, count # store it back + +# release lock +mov $0, mutex + +# see if we're still looping +sub $1, %bx +test $0, %bx +jgt .top + +halt diff --git a/related_info/ostep/ostep12-threadlock/test-and-test-and-set.s b/related_info/ostep/ostep12-threadlock/test-and-test-and-set.s new file mode 100644 index 0000000..8c225b7 --- /dev/null +++ b/related_info/ostep/ostep12-threadlock/test-and-test-and-set.s @@ -0,0 +1,29 @@ +.var mutex +.var count + +.main +.top + +.acquire +mov mutex, %ax +test $0, %ax +jne .acquire +mov $1, %ax +xchg %ax, mutex # atomic swap of 1 and mutex +test $0, %ax # if we get 0 back: lock is free! +jne .acquire # if not, try again + +# critical section +mov count, %ax # get the value at the address +add $1, %ax # increment it +mov %ax, count # store it back + +# release lock +mov $0, mutex + +# see if we're still looping +sub $1, %bx +test $0, %bx +jgt .top + +halt diff --git a/related_info/ostep/ostep12-threadlock/ticket.s b/related_info/ostep/ostep12-threadlock/ticket.s new file mode 100644 index 0000000..9c91ea9 --- /dev/null +++ b/related_info/ostep/ostep12-threadlock/ticket.s @@ -0,0 +1,30 @@ +.var ticket +.var turn +.var count + +.main +.top + +.acquire +mov $1, %ax +fetchadd %ax, ticket # grab a ticket (keep it in dx) +.tryagain +mov turn, %cx # check if it's your turn +test %cx, %ax +jne .tryagain + +# critical section +mov count, %ax # get the value at the address +add $1, %ax # increment it +mov %ax, count # store it back + +# release lock +mov $1, %ax +fetchadd %ax, turn + +# see if we're still looping +sub $1, %bx +test $0, %bx +jgt .top + +halt diff --git a/related_info/ostep/ostep12-threadlock/x86.py b/related_info/ostep/ostep12-threadlock/x86.py new file mode 100755 index 0000000..aec6452 --- /dev/null +++ b/related_info/ostep/ostep12-threadlock/x86.py @@ -0,0 +1,1186 @@ +#! /usr/bin/env python + +import sys +import time +import random +from optparse import OptionParser + +# +# HELPER +# +def dospace(howmuch): + for i in range(howmuch): + print '%24s' % ' ', + +# useful instead of assert +def zassert(cond, str): + if cond == False: + print 'ABORT::', str + exit(1) + return + +class cpu: + # + # INIT: how much memory? + # + def __init__(self, memory, memtrace, regtrace, cctrace, compute, verbose, printstats, headercount): + # + # CONSTANTS + # + + # conditions + self.COND_GT = 0 + self.COND_GTE = 1 + self.COND_LT = 2 + self.COND_LTE = 3 + self.COND_EQ = 4 + self.COND_NEQ = 5 + + # registers in system + self.REG_ZERO = 0 + self.REG_AX = 1 + self.REG_BX = 2 + self.REG_CX = 3 + self.REG_DX = 4 + self.REG_EX = 5 + self.REG_FX = 6 + self.REG_SP = 7 + self.REG_BP = 8 + + # system memory: in KB + self.max_memory = memory * 1024 + + # which memory addrs and registers to trace? + self.memtrace = memtrace + self.regtrace = regtrace + self.cctrace = cctrace + self.compute = compute + self.verbose = verbose + self.printstats = printstats + self.headercount = headercount + + self.PC = 0 + self.registers = {} + self.conditions = {} + self.labels = {} + self.vars = {} + self.memory = {} + self.pmemory = {} # for printable version of what's in memory (instructions) + + self.condlist = [self.COND_GTE, self.COND_GT, self.COND_LTE, + self.COND_LT, self.COND_NEQ, self.COND_EQ] + self.regnums = [self.REG_ZERO, + self.REG_AX, self.REG_BX, self.REG_CX, + self.REG_DX, self.REG_EX, self.REG_FX, + self.REG_SP, self.REG_BP] + + self.regnames = {} + self.regnames['zero'] = self.REG_ZERO # hidden zero-valued register + self.regnames['ax'] = self.REG_AX + self.regnames['bx'] = self.REG_BX + self.regnames['cx'] = self.REG_CX + self.regnames['dx'] = self.REG_DX + self.regnames['ex'] = self.REG_EX + self.regnames['fx'] = self.REG_FX + self.regnames['sp'] = self.REG_SP + self.regnames['bp'] = self.REG_BP + + tmplist = [] + for r in self.regtrace: + zassert(r in self.regnames, 'Register %s cannot be traced because it does not exist' % r) + tmplist.append(self.regnames[r]) + self.regtrace = tmplist + + self.init_memory() + self.init_registers() + self.init_condition_codes() + + # + # BEFORE MACHINE RUNS + # + def init_condition_codes(self): + for c in self.condlist: + self.conditions[c] = False + + def init_memory(self): + for i in range(self.max_memory): + self.memory[i] = 0 + + def init_registers(self): + for i in self.regnums: + self.registers[i] = 0 + + def dump_memory(self): + print 'MEMORY DUMP' + for i in range(self.max_memory): + if i not in self.pmemory and i in self.memory and self.memory[i] != 0: + print ' m[%d]' % i, self.memory[i] + + # + # INFORMING ABOUT THE HARDWARE + # + def get_regnum(self, name): + assert(name in self.regnames) + return self.regnames[name] + + def get_regname(self, num): + assert(num in self.regnums) + for rname in self.regnames: + if self.regnames[rname] == num: + return rname + return '' + + def get_regnums(self): + return self.regnums + + def get_condlist(self): + return self.condlist + + def get_reg(self, reg): + assert(reg in self.regnums) + return self.registers[reg] + + def get_cond(self, cond): + assert(cond in self.condlist) + return self.conditions[cond] + + def get_pc(self): + return self.PC + + def set_reg(self, reg, value): + assert(reg in self.regnums) + self.registers[reg] = value + + def set_cond(self, cond, value): + assert(cond in self.condlist) + self.conditions[cond] = value + + def set_pc(self, pc): + self.PC = pc + + # + # INSTRUCTIONS + # + def halt(self): + return -1 + + def iyield(self): + return -2 + + def nop(self): + return 0 + + def rdump(self): + print 'REGISTERS::', + print 'ax:', self.registers[self.REG_AX], + print 'bx:', self.registers[self.REG_BX], + print 'cx:', self.registers[self.REG_CX], + print 'dx:', self.registers[self.REG_DX], + print 'ex:', self.registers[self.REG_EX], + print 'fx:', self.registers[self.REG_FX], + return + + def mdump(self, index): + print 'm[%d] ' % index, self.memory[index] + return + + # + # MEMORY MOVES + # + def move_i_to_r(self, src, dst): + self.registers[dst] = src + return 0 + + # memory: value, register, register + def move_i_to_m(self, src, value, reg1, reg2, scale): + tmp = value + self.registers[reg1] + (scale * self.registers[reg2]) + self.memory[tmp] = src + return 0 + + def move_m_to_r(self, value, reg1, reg2, scale, dst): + tmp = value + self.registers[reg1] + (scale * self.registers[reg2]) + self.registers[dst] = self.memory[tmp] + + def move_r_to_m(self, src, value, reg1, reg2, scale): + tmp = value + self.registers[reg1] + (scale * self.registers[reg2]) + self.memory[tmp] = self.registers[src] + return 0 + + def move_r_to_r(self, src, dst): + self.registers[dst] = self.registers[src] + return 0 + + # + # LOAD EFFECTIVE ADDRESS (everything but the final change of memory value) + # + def lea_m_to_r(self, value, reg1, reg2, scale, dst): + tmp = value + self.registers[reg1] + (scale * self.registers[reg2]) + self.registers[dst] = tmp + + # + # ARITHMETIC INSTRUCTIONS + # + def add_i_to_r(self, src, dst): + self.registers[dst] += src + return 0 + + def add_r_to_r(self, src, dst): + self.registers[dst] += self.registers[src] + return 0 + + def mul_i_to_r(self, src, dst): + self.registers[dst] *= src + return 0 + + def mul_r_to_r(self, src, dst): + self.registers[dst] *= self.registers[src] + return 0 + + def sub_i_to_r(self, src, dst): + self.registers[dst] -= src + return 0 + + def sub_r_to_r(self, src, dst): + self.registers[dst] -= self.registers[src] + return 0 + + def neg_r(self, src): + self.registers[src] = -self.registers[src] + + # + # SUPPORT FOR LOCKS + # + def atomic_exchange(self, src, value, reg1, reg2): + tmp = value + self.registers[reg1] + self.registers[reg2] + old = self.memory[tmp] + self.memory[tmp] = self.registers[src] + self.registers[src] = old + return 0 + + def fetchadd(self, src, value, reg1, reg2): + tmp = value + self.registers[reg1] + self.registers[reg2] + old = self.memory[tmp] + self.memory[tmp] = self.memory[tmp] + self.registers[src] + self.registers[src] = old + + # + # TEST for conditions + # + def test_all(self, src, dst): + self.init_condition_codes() + if dst > src: + self.conditions[self.COND_GT] = True + if dst >= src: + self.conditions[self.COND_GTE] = True + if dst < src: + self.conditions[self.COND_LT] = True + if dst <= src: + self.conditions[self.COND_LTE] = True + if dst == src: + self.conditions[self.COND_EQ] = True + if dst != src: + self.conditions[self.COND_NEQ] = True + return 0 + + def test_i_r(self, src, dst): + self.init_condition_codes() + return self.test_all(src, self.registers[dst]) + + def test_r_i(self, src, dst): + self.init_condition_codes() + return self.test_all(self.registers[src], dst) + + def test_r_r(self, src, dst): + self.init_condition_codes() + return self.test_all(self.registers[src], self.registers[dst]) + + # + # JUMPS + # + def jump(self, targ): + self.PC = targ + return 0 + + def jump_notequal(self, targ): + if self.conditions[self.COND_NEQ] == True: + self.PC = targ + return 0 + + def jump_equal(self, targ): + if self.conditions[self.COND_EQ] == True: + self.PC = targ + return 0 + + def jump_lessthan(self, targ): + if self.conditions[self.COND_LT] == True: + self.PC = targ + return 0 + + def jump_lessthanorequal(self, targ): + if self.conditions[self.COND_LTE] == True: + self.PC = targ + return 0 + + def jump_greaterthan(self, targ): + if self.conditions[self.COND_GT] == True: + self.PC = targ + return 0 + + def jump_greaterthanorequal(self, targ): + if self.conditions[self.COND_GTE] == True: + self.PC = targ + return 0 + + # + # CALL and RETURN + # + def call(self, targ): + self.registers[self.REG_SP] -= 4 + self.memory[self.registers[self.REG_SP]] = self.PC + self.PC = targ + + def ret(self): + self.PC = self.memory[self.registers[self.REG_SP]] + self.registers[self.REG_SP] += 4 + + # + # STACK and related + # + def push_r(self, reg): + self.registers[self.REG_SP] -= 4 + self.memory[self.registers[self.REG_SP]] = self.registers[reg] + return 0 + + def push_m(self, value, reg1, reg2, scale): + self.registers[self.REG_SP] -= 4 + tmp = value + self.registers[reg1] + (self.registers[reg2] * scale) + # push address onto stack, not memory value itself + self.memory[self.registers[self.REG_SP]] = tmp + return 0 + + def pop(self): + self.registers[self.REG_SP] += 4 + + def pop_r(self, dst): + self.registers[dst] = self.memory[self.registers[self.REG_SP]] + self.registers[self.REG_SP] += 4 + + # + # HELPER func for getarg + # + def register_translate(self, r): + if r in self.regnames: + return self.regnames[r] + zassert(False, 'Register %s is not a valid register' % r) + return + + def getregname(self, r): + t = r.strip() + if t == '': + return 'zero' + zassert(t[0] == '%', 'Expecting a proper register name, got [%s]' % r) + return r.split('%')[1].strip() + + # + # HELPER in parsing mov (quite primitive) and other ops + # returns: (value, type) + # where type is (TYPE_REGISTER, TYPE_IMMEDIATE, TYPE_MEMORY) + # + # FORMATS + # %ax - register + # $10 - immediate + # 10 - direct memory + # 10(%ax) - memory + reg indirect + # 10(%ax,%bx) - memory + 2 reg indirect + # 10(%ax,%bx,4) - XXX (not handled) + # + def getarg(self, arg): + tmp1 = arg.replace(',', ' ') + tmp = tmp1.replace(' \t', '') + + if tmp[0] == '$': + # this is an IMMEDIATE VALUE + value = tmp.split('$')[1] + neg = 1 + if value[0] == '-': + value = value[1:] + neg = -1 + zassert(value.isdigit(), 'value [%s] must be a digit' % value) + return neg * int(value), 'TYPE_IMMEDIATE' + elif tmp[0] == '%': + # this is a REGISTER + register = tmp.split('%')[1] + return self.register_translate(register), 'TYPE_REGISTER' + elif tmp[0] == '.': + # this is a LABEL + targ = tmp + return targ, 'TYPE_LABEL' + elif tmp[0].isalpha() and not tmp[0].isdigit(): + # this is a VARIABLE + zassert(tmp in self.vars, 'Variable %s is not declared' % tmp) + return '%d,%d,%d,1' % (self.vars[tmp], self.register_translate('zero'), self.register_translate('zero')), 'TYPE_MEMORY' + elif tmp[0].isdigit() or tmp[0] == '-' or tmp[0] == '(': + # MOST GENERAL CASE: number(reg,reg) or number(reg) or number(reg,reg,number) + neg = 1 + if tmp[0] == '-': + tmp = tmp[1:] + neg = -1 + s = tmp.split('(') + if len(s) == 1: + # no parens -> we just assume that we have a constant value (an address), e.g., mov 10, %ax + value = neg * int(tmp) + return '%d,%d,%d,1' % (int(value), self.register_translate('zero'), self.register_translate('zero')), 'TYPE_MEMORY' + elif len(s) == 2: + # here we just assume that we have something in parentheses + # e.g., mov 10(%ax) or mov 10(%ax,%bx) or mov 10(%ax,%bx,10) or mov (%ax,%bx,10) or ... + + # if no leading number exists, first char should be a paren; in that case, value is just made to be 0 + # otherwise we should handle either a number or a negative number + if tmp[0] != '(': + zassert(s[0].strip().isdigit() == True, 'First number should be a digit [%s]' % s[0]) + value = neg * int(s[0]) + else: + value = 0 + t = s[1].split(')')[0].split('__BREAK__') + if len(t) == 1: + register = self.getregname(t[0]) + return '%d,%d,%d,1' % (int(value), self.register_translate(register), self.register_translate('zero')), 'TYPE_MEMORY' + elif len(t) == 2: + register1 = self.getregname(t[0]) + register2 = self.getregname(t[1]) + return '%d,%d,%d,1' % (int(value), self.register_translate(register1), self.register_translate(register2)), 'TYPE_MEMORY' + elif len(t) == 3: + register1 = self.getregname(t[0]) + register2 = self.getregname(t[1]) + scale = int(t[2]) + return '%d,%d,%d,%d' % (int(value), self.register_translate(register1), self.register_translate(register2), scale), 'TYPE_MEMORY' + else: + print 'mov: bad argument [%s]' % tmp + exit(1) + return + else: + print 'mov: bad argument [%s]' % tmp + exit(1) + return + zassert(True, 'mov: bad argument [%s]' % arg) + return + + # + # helper function in parsing complex args to mov/lea instruction + # + def removecommas(self, cline, inargs): + inparen = False + outargs = '' + for i in range(len(inargs)): + if inargs[i] == '(': + zassert(inparen == False, 'cannot have nested parenthesis in argument [%s]' % cline) + inparen = True + if inargs[i] == ')': + zassert(inparen == True, 'cannot have right parenthesis without first having left one [%s]' % cline) + inparen = False + if inparen == True: + if inargs[i] == ',': + outargs += '__BREAK__' + else: + outargs += inargs[i] + else: + outargs += inargs[i] + zassert(inparen == False, 'did not close parentheses [%s]' % cline) + return outargs + + # + # LOAD a program into memory + # make it ready to execute + # + def load(self, infile, loadaddr): + pc = int(loadaddr) + fd = open(infile) + + bpc = loadaddr + data = 100 + + for line in fd: + cline = line.rstrip() + + # remove everything after the comment marker + ctmp = cline.split('#') + assert(len(ctmp) == 1 or len(ctmp) == 2) + if len(ctmp) == 2: + cline = ctmp[0] + + # remove empty lines, and split line by spaces + tmp = cline.split() + if len(tmp) == 0: + continue + + # only pay attention to labels and variables + if tmp[0] == '.var': + assert(len(tmp) == 2 or len(tmp) == 3) + assert(tmp[0] not in self.vars) + self.vars[tmp[1]] = data + mul = 1 + if len(tmp) == 3: + mul = int(tmp[2]) + data += (4 * mul) + zassert(data < bpc, 'Load address overrun by static data') + if self.verbose: print 'ASSIGN VAR', tmp[0], "-->", tmp[1], self.vars[tmp[1]] + elif tmp[0][0] == '.': + assert(len(tmp) == 1) + self.labels[tmp[0]] = int(pc) + if self.verbose: print 'ASSIGN LABEL', tmp[0], "-->", pc + else: + pc += 1 + fd.close() + + if self.verbose: print '' + + # second pass: do everything else + pc = int(loadaddr) + fd = open(infile) + for line in fd: + cline = line.rstrip() + + # remove everything after the comment marker + ctmp = cline.split('#') + assert(len(ctmp) == 1 or len(ctmp) == 2) + if len(ctmp) == 2: + cline = ctmp[0] + + # remove empty lines, and split line by spaces + tmp = cline.split() + if len(tmp) == 0: + continue + + # skip labels: all else must be instructions + if cline[0] != '.': + tmp = cline.split(None, 1) + opcode = tmp[0] + self.pmemory[pc] = cline.strip() + + if self.verbose == True: + print 'opcode', opcode + + # MAIN OPCODE LOOP + if opcode == 'mov': + # most painful one to parse (due to generic form) + # could be mov x(r1,r2,4), r3 or mov r1, (r2,r3) or ... + outargs = self.removecommas(cline, tmp[1]) + + rtmp = outargs.split(',') + zassert(len(rtmp) == 2, 'mov: needs two args, separated by commas [%s]' % cline) + arg1 = rtmp[0].strip() + arg2 = rtmp[1].strip() + (src, stype) = self.getarg(arg1) + (dst, dtype) = self.getarg(arg2) + # print 'MOV', src, stype, dst, dtype + if stype == 'TYPE_MEMORY' and dtype == 'TYPE_MEMORY': + print 'bad mov: two memory arguments' + exit(1) + elif stype == 'TYPE_IMMEDIATE' and dtype == 'TYPE_IMMEDIATE': + print 'bad mov: two immediate arguments' + exit(1) + elif stype == 'TYPE_IMMEDIATE' and dtype == 'TYPE_REGISTER': + self.memory[pc] = 'self.move_i_to_r(%d, %d)' % (int(src), dst) + elif stype == 'TYPE_IMMEDIATE' and dtype == 'TYPE_REGISTER': + self.memory[pc] = 'self.move_i_to_r(%d, %d)' % (int(src), dst) + elif stype == 'TYPE_MEMORY' and dtype == 'TYPE_REGISTER': + tmp = src.split(',') + assert(len(tmp) == 4) + self.memory[pc] = 'self.move_m_to_r(%d, %d, %d, %d, %d)' % (int(tmp[0]), int(tmp[1]), int(tmp[2]), int(tmp[3]), dst) + elif stype == 'TYPE_REGISTER' and dtype == 'TYPE_MEMORY': + tmp = dst.split(',') + assert(len(tmp) == 4) + self.memory[pc] = 'self.move_r_to_m(%d, %d, %d, %d, %d)' % (src, int(tmp[0]), int(tmp[1]), int(tmp[2]), int(tmp[3])) + elif stype == 'TYPE_REGISTER' and dtype == 'TYPE_REGISTER': + self.memory[pc] = 'self.move_r_to_r(%d, %d)' % (src, dst) + elif stype == 'TYPE_IMMEDIATE' and dtype == 'TYPE_MEMORY': + tmp = dst.split(',') + assert(len(tmp) == 4) + self.memory[pc] = 'self.move_i_to_m(%d, %d, %d, %d, %d)' % (src, int(tmp[0]), int(tmp[1]), int(tmp[2]), int(tmp[3])) + else: + zassert(False, 'malformed mov instruction') + elif opcode == 'lea': + rtmp = tmp[1].split(',', 1) + zassert(len(tmp) == 2 and len(rtmp) == 2, 'lea: needs two args, separated by commas [%s]' % cline) + arg1 = rtmp[0].strip() + arg2 = rtmp[1].strip() + (src, stype) = self.getarg(arg1) + (dst, dtype) = self.getarg(arg2) + if stype == 'TYPE_MEMORY' and dtype == 'TYPE_REGISTER': + tmp = src.split(',') + assert(len(tmp) == 4) + self.memory[pc] = 'self.lea_m_to_r(%d, %d, %d, %d, %d)' % (int(tmp[0]), int(tmp[1]), int(tmp[2]), int(tmp[3]), dst) + else: + zassert(False, 'malformed lea instruction (should be memory address source to register destination') + elif opcode == 'neg': + zassert(len(tmp) == 2, 'neg: takes one argument') + arg = tmp[1].strip() + (dst, dtype) = self.getarg(arg) + zassert(dtype == 'TYPE_REGISTER', 'Can only neg a register') + self.memory[pc] = 'self.neg_r(%d)' % dst + elif opcode == 'pop': + if len(tmp) == 1: + self.memory[pc] = 'self.pop()' + elif len(tmp) == 2: + arg = tmp[1].strip() + (dst, dtype) = self.getarg(arg) + zassert(dtype == 'TYPE_REGISTER', 'Can only pop into a register') + self.memory[pc] = 'self.pop_r(%d)' % dst + else: + zassert(False, 'pop instruction must take zero/one args') + elif opcode == 'push': + (src, stype) = self.getarg(tmp[1].strip()) + if stype == 'TYPE_REGISTER': + self.memory[pc] = 'self.push_r(%d)' % (int(src)) + elif stype == 'TYPE_MEMORY': + tmp = src.split(',') + assert(len(tmp) == 4) + self.memory[pc] = 'self.push_m(%d,%d,%d,%d)' % (int(tmp[0]), int(tmp[1]), int(tmp[2]), int(tmp[3])) + else: + zassert(False, 'Cannot push anything but registers') + elif opcode == 'call': + (targ, ttype) = self.getarg(tmp[1].strip()) + if ttype == 'TYPE_LABEL': + self.memory[pc] = 'self.call(%d)' % (int(self.labels[targ])) + else: + zassert(False, 'Cannot call anything but a label') + elif opcode == 'ret': + assert(len(tmp) == 1) + self.memory[pc] = 'self.ret()' + elif opcode == 'mul': + rtmp = tmp[1].split(',', 1) + zassert(len(tmp) == 2 and len(rtmp) == 2, 'mul: needs two args, separated by commas [%s]' % cline) + arg1 = rtmp[0].strip() + arg2 = rtmp[1].strip() + (src, stype) = self.getarg(arg1) + (dst, dtype) = self.getarg(arg2) + if stype == 'TYPE_IMMEDIATE' and dtype == 'TYPE_REGISTER': + self.memory[pc] = 'self.mul_i_to_r(%d, %d)' % (int(src), dst) + elif stype == 'TYPE_REGISTER' and dtype == 'TYPE_REGISTER': + self.memory[pc] = 'self.mul_r_to_r(%d, %d)' % (int(src), dst) + else: + zassert(False, 'malformed usage of add instruction') + elif opcode == 'add': + rtmp = tmp[1].split(',', 1) + zassert(len(tmp) == 2 and len(rtmp) == 2, 'add: needs two args, separated by commas [%s]' % cline) + arg1 = rtmp[0].strip() + arg2 = rtmp[1].strip() + (src, stype) = self.getarg(arg1) + (dst, dtype) = self.getarg(arg2) + if stype == 'TYPE_IMMEDIATE' and dtype == 'TYPE_REGISTER': + self.memory[pc] = 'self.add_i_to_r(%d, %d)' % (int(src), dst) + elif stype == 'TYPE_REGISTER' and dtype == 'TYPE_REGISTER': + self.memory[pc] = 'self.add_r_to_r(%d, %d)' % (int(src), dst) + else: + zassert(False, 'malformed usage of add instruction') + elif opcode == 'sub': + rtmp = tmp[1].split(',', 1) + zassert(len(tmp) == 2 and len(rtmp) == 2, 'sub: needs two args, separated by commas [%s]' % cline) + arg1 = rtmp[0].strip() + arg2 = rtmp[1].strip() + (src, stype) = self.getarg(arg1) + (dst, dtype) = self.getarg(arg2) + if stype == 'TYPE_IMMEDIATE' and dtype == 'TYPE_REGISTER': + self.memory[pc] = 'self.sub_i_to_r(%d, %d)' % (int(src), dst) + elif stype == 'TYPE_REGISTER' and dtype == 'TYPE_REGISTER': + self.memory[pc] = 'self.sub_r_to_r(%d, %d)' % (int(src), dst) + else: + zassert(False, 'malformed usage of sub instruction') + elif opcode == 'fetchadd': + rtmp = tmp[1].split(',', 1) + zassert(len(tmp) == 2 and len(rtmp) == 2, 'fetchadd: needs two args, separated by commas [%s]' % cline) + arg1 = rtmp[0].strip() + arg2 = rtmp[1].strip() + (src, stype) = self.getarg(arg1) + (dst, dtype) = self.getarg(arg2) + tmp = dst.split(',') + assert(len(tmp) == 4) + if stype == 'TYPE_REGISTER' and dtype == 'TYPE_MEMORY': + self.memory[pc] = 'self.fetchadd(%d, %d, %d, %d)' % (src, int(tmp[0]), int(tmp[1]), int(tmp[2])) + else: + zassert(False, 'poorly specified fetch and add') + elif opcode == 'xchg': + rtmp = tmp[1].split(',', 1) + zassert(len(tmp) == 2 and len(rtmp) == 2, 'xchg: needs two args, separated by commas [%s]' % cline) + arg1 = rtmp[0].strip() + arg2 = rtmp[1].strip() + (src, stype) = self.getarg(arg1) + (dst, dtype) = self.getarg(arg2) + tmp = dst.split(',') + assert(len(tmp) == 4) + if stype == 'TYPE_REGISTER' and dtype == 'TYPE_MEMORY': + self.memory[pc] = 'self.atomic_exchange(%d, %d, %d, %d)' % (src, int(tmp[0]), int(tmp[1]), int(tmp[2])) + else: + zassert(False, 'poorly specified atomic exchange') + elif opcode == 'test': + rtmp = tmp[1].split(',', 1) + zassert(len(tmp) == 2 and len(rtmp) == 2, 'test: needs two args, separated by commas [%s]' % cline) + arg1 = rtmp[0].strip() + arg2 = rtmp[1].strip() + (src, stype) = self.getarg(arg1) + (dst, dtype) = self.getarg(arg2) + if stype == 'TYPE_IMMEDIATE' and dtype == 'TYPE_REGISTER': + self.memory[pc] = 'self.test_i_r(%d, %d)' % (int(src), dst) + elif stype == 'TYPE_REGISTER' and dtype == 'TYPE_REGISTER': + self.memory[pc] = 'self.test_r_r(%d, %d)' % (int(src), dst) + elif stype == 'TYPE_REGISTER' and dtype == 'TYPE_IMMEDIATE': + self.memory[pc] = 'self.test_r_i(%d, %d)' % (int(src), dst) + else: + zassert(False, 'malformed usage of test instruction') + elif opcode == 'j': + (targ, ttype) = self.getarg(tmp[1].strip()) + zassert(ttype == 'TYPE_LABEL', 'bad jump target [%s]' % tmp[1].strip()) + self.memory[pc] = 'self.jump(%d)' % int(self.labels[targ]) + elif opcode == 'jne': + (targ, ttype) = self.getarg(tmp[1].strip()) + zassert(ttype == 'TYPE_LABEL', 'bad jump target [%s]' % tmp[1].strip()) + self.memory[pc] = 'self.jump_notequal(%d)' % int(self.labels[targ]) + elif opcode == 'je': + (targ, ttype) = self.getarg(tmp[1].strip()) + zassert(ttype == 'TYPE_LABEL', 'bad jump target [%s]' % tmp[1].strip()) + self.memory[pc] = 'self.jump_equal(%d)' % self.labels[targ] + elif opcode == 'jlt': + (targ, ttype) = self.getarg(tmp[1].strip()) + zassert(ttype == 'TYPE_LABEL', 'bad jump target [%s]' % tmp[1].strip()) + self.memory[pc] = 'self.jump_lessthan(%d)' % int(self.labels[targ]) + elif opcode == 'jlte': + (targ, ttype) = self.getarg(tmp[1].strip()) + zassert(ttype == 'TYPE_LABEL', 'bad jump target [%s]' % tmp[1].strip()) + self.memory[pc] = 'self.jump_lessthanorequal(%s)' % self.labels[targ] + elif opcode == 'jgt': + (targ, ttype) = self.getarg(tmp[1].strip()) + zassert(ttype == 'TYPE_LABEL', 'bad jump target [%s]' % tmp[1].strip()) + self.memory[pc] = 'self.jump_greaterthan(%d)' % int(self.labels[targ]) + elif opcode == 'jgte': + (targ, ttype) = self.getarg(tmp[1].strip()) + zassert(ttype == 'TYPE_LABEL', 'bad jump target [%s]' % tmp[1].strip()) + self.memory[pc] = 'self.jump_greaterthanorequal(%s)' % self.labels[targ] + elif opcode == 'nop': + self.memory[pc] = 'self.nop()' + elif opcode == 'halt': + self.memory[pc] = 'self.halt()' + elif opcode == 'yield': + self.memory[pc] = 'self.iyield()' + elif opcode == 'rdump': + self.memory[pc] = 'self.rdump()' + elif opcode == 'mdump': + self.memory[pc] = 'self.mdump(%s)' % tmp[1] + else: + print 'illegal opcode: ', opcode + exit(1) + + if self.verbose: print 'pc:%d LOADING %20s --> %s' % (pc, self.pmemory[pc], self.memory[pc]) + + # INCREMENT PC for loader + pc += 1 + # END: loop over file + fd.close() + if self.verbose: print '' + return + # END: load + + def print_headers(self, procs): + # print some headers + if self.printstats == True: + print 'icount', + if len(self.memtrace) > 0: + for m in self.memtrace: + if m[0].isdigit(): + print '%5d' % int(m), + else: + zassert(m in self.vars, 'Traced variable %s not declared' % m) + print '%5s' % m, + print ' ', + if len(self.regtrace) > 0: + for r in self.regtrace: + print '%5s' % self.get_regname(r), + print ' ', + if cctrace == True: + print '>= > <= < != ==', + + # and per thread + for i in range(procs.getnum()): + print ' Thread %d ' % i, + print '' + return + + def print_trace(self, newline): + if self.printstats == True: + print '%6d' % self.icount, + if len(self.memtrace) > 0: + for m in self.memtrace: + if self.compute: + if m[0].isdigit(): + print '%5d' % self.memory[int(m)], + else: + zassert(m in self.vars, 'Traced variable %s not declared' % m) + print '%5d' % self.memory[self.vars[m]], + else: + print '%5s' % '?', + print ' ', + if len(self.regtrace) > 0: + for r in self.regtrace: + if self.compute: + print '%5d' % self.registers[r], + else: + print '%5s' % '?', + print ' ', + if cctrace == True: + for c in self.condlist: + if self.compute: + if self.conditions[c]: + print '1 ', + else: + print '0 ', + else: + print '? ', + if (len(self.memtrace) > 0 or len(self.regtrace) > 0 or cctrace == True) and newline == True: + print '' + return + + def setint(self, intfreq, intrand): + if intrand == False: + return intfreq + return int(random.random() * intfreq) + 1 + + def run(self, procs, intfreq, intrand): + # hw init: cc's, interrupt frequency, etc. + if procs.ismanual() == True: + intfreq = 1 + interrupt = 1 + intrand = False + + interrupt = self.setint(intfreq, intrand) + self.icount = 0 + + # you always get one printing + print '' + self.print_headers(procs) + print '' + self.print_trace(True) + + while True: + if self.headercount > 0 and self.icount % self.headercount == 0 and self.icount > 0: + print '' + self.print_headers(procs) + print '' + self.print_trace(True) + + # need thread ID of current process + tid = procs.getcurr().gettid() + + # FETCH + prevPC = self.PC + instruction = self.memory[self.PC] + self.PC += 1 + + # DECODE and EXECUTE + # key: self.PC may be changed during eval; thus MUST be incremented BEFORE eval + rc = eval(instruction) + + # tracing details: ALWAYS AFTER EXECUTION OF INSTRUCTION + self.print_trace(False) + + # output: thread-proportional spacing followed by PC and instruction + dospace(tid) + print prevPC, self.pmemory[prevPC] + self.icount += 1 + + # halt instruction issued + if rc == -1: + procs.done() + # finish execution by returning from run() + if procs.numdone() == procs.getnum(): + return self.icount + procs.next() + procs.restore() + + self.print_trace(False) + for i in range(procs.getnum()): + print '----- Halt;Switch ----- ', + print '' + + # do interrupt processing + # just counts down the interrupt counter to zero + # when it gets to 0, or when the 'yield' instruction is issued (rc=-2) + # a switch takes place + # key thing: if manual scheduling is done (procsched), interrupt + # must take place every instruction for this to work + interrupt -= 1 + if interrupt == 0 or rc == -2: + interrupt = self.setint(intfreq, intrand) + curr = procs.getcurr() + procs.save() + procs.next() + procs.restore() + next = procs.getcurr() + + if procs.ismanual() == False or (procs.ismanual() == True and curr != next): + self.print_trace(False) + for i in range(procs.getnum()): + print '------ Interrupt ------ ', + print '' + + # END: while + return + +# +# END: class cpu +# + + +# +# PROCESS LIST class +# +# Tracks all running processes in the program +# Also deals with manual scheduling as specified by user +# +class proclist: + def __init__(self): + self.plist = [] # list of process objects + self.active = 0 # tracks how many processes are active + self.manual = False + self.procsched = [] # list of which processes to run in what order (by ID) + self.curr = 0 # currently running process (index into procsched list) + + def finalize(self, procsched): + if procsched == '': + for i in range(len(self.plist)): + self.procsched.append(i) + self.curr = 0 + self.restore() + return + + # in this case, user has passed in schedule + self.manual = True + for i in range(len(procsched)): + p = int(procsched[i]) + if p >= self.getnum(): + print 'bad schedule: cannot include a thread that does not exist (%d)' % p + exit(1) + self.procsched.append(p) + check = [] + for p in self.procsched: + if p not in check: + check.append(p) + if len(check) != self.active: + print 'bad schedule: does not include ALL processes', self.procsched + exit(1) + self.curr = 0 + self.restore() + return + + def addproc(self, p): + self.active += 1 + self.plist.append(p) + return + + def ismanual(self): + return self.manual + + def done(self): + p = self.procsched[self.curr] + self.plist[p].setdone() + self.active -= 1 + return + + def numdone(self): + return len(self.plist) - self.active + + def getnum(self): + return len(self.plist) + + def getcurr(self): + return self.plist[self.procsched[self.curr]] + + def save(self): + self.plist[self.procsched[self.curr]].save() + return + + def restore(self): + self.plist[self.procsched[self.curr]].restore() + return + + def next(self): + while True: + self.curr += 1 + if self.curr == len(self.procsched): + self.curr = 0 + p = self.procsched[self.curr] + if self.plist[p].isdone() == False: + return + return + + +# +# PROCESS class +# +class process: + def __init__(self, cpu, tid, pc, stackbottom, reginit): + self.cpu = cpu # object reference + self.tid = tid + self.pc = pc + self.regs = {} + self.cc = {} + self.done = False + self.stack = stackbottom + + # init regs: all 0 or specially set to something + for r in self.cpu.get_regnums(): + self.regs[r] = 0 + if reginit != '': + # form: ax=1,bx=2 (for some subset of registers) + for r in reginit.split(':'): + tmp = r.split('=') + assert(len(tmp) == 2) + self.regs[self.cpu.get_regnum(tmp[0])] = int(tmp[1]) + + # init CCs + for c in self.cpu.get_condlist(): + self.cc[c] = False + + # stack + self.regs[self.cpu.get_regnum('sp')] = stackbottom + + return + + def gettid(self): + return self.tid + + def save(self): + self.pc = self.cpu.get_pc() + for c in self.cpu.get_condlist(): + self.cc[c] = self.cpu.get_cond(c) + for r in self.cpu.get_regnums(): + self.regs[r] = self.cpu.get_reg(r) + + def restore(self): + self.cpu.set_pc(self.pc) + for c in self.cpu.get_condlist(): + self.cpu.set_cond(c, self.cc[c]) + for r in self.cpu.get_regnums(): + self.cpu.set_reg(r, self.regs[r]) + + def setdone(self): + self.done = True + + def isdone(self): + return self.done == True + +# +# main program +# +parser = OptionParser() +parser.add_option('-s', '--seed', default=0, help='the random seed', action='store', type='int', dest='seed') +parser.add_option('-t', '--threads', default=2, help='number of threads', action='store', type='int', dest='numthreads') +parser.add_option('-p', '--program', default='', help='source program (in .s)', action='store', type='string', dest='progfile') +parser.add_option('-i', '--interrupt', default=50, help='interrupt frequency', action='store', type='int', dest='intfreq') +parser.add_option('-P', '--procsched', default='', help='control exactly which thread runs when', + action='store', type='string', dest='procsched') +parser.add_option('-r', '--randints', default=False, help='if interrupts are random', action='store_true', dest='intrand') +parser.add_option('-a', '--argv', default='', + help='comma-separated per-thread args (e.g., ax=1,ax=2 sets thread 0 ax reg to 1 and thread 1 ax reg to 2); specify multiple regs per thread via colon-separated list (e.g., ax=1:bx=2,cx=3 sets thread 0 ax and bx and just cx for thread 1)', + action='store', type='string', dest='argv') +parser.add_option('-L', '--loadaddr', default=1000, help='address where to load code', action='store', type='int', dest='loadaddr') +parser.add_option('-m', '--memsize', default=128, help='size of address space (KB)', action='store', type='int', dest='memsize') +parser.add_option('-M', '--memtrace', default='', help='comma-separated list of addrs to trace (e.g., 20000,20001)', action='store', + type='string', dest='memtrace') +parser.add_option('-R', '--regtrace', default='', help='comma-separated list of regs to trace (e.g., ax,bx,cx,dx)', action='store', + type='string', dest='regtrace') +parser.add_option('-C', '--cctrace', default=False, help='should we trace condition codes', action='store_true', dest='cctrace') +parser.add_option('-S', '--printstats',default=False, help='print some extra stats', action='store_true', dest='printstats') +parser.add_option('-v', '--verbose', default=False, help='print some extra info', action='store_true', dest='verbose') +parser.add_option('-H', '--headercount',default=-1, help='how often to print a row header', action='store', type='int', dest='headercount') +parser.add_option('-c', '--compute', default=False, help='compute answers for me', action='store_true', dest='solve') +(options, args) = parser.parse_args() + +print 'ARG seed', options.seed +print 'ARG numthreads', options.numthreads +print 'ARG program', options.progfile +print 'ARG interrupt frequency', options.intfreq +print 'ARG interrupt randomness',options.intrand +print 'ARG procsched', options.procsched +print 'ARG argv', options.argv +print 'ARG load address', options.loadaddr +print 'ARG memsize', options.memsize +print 'ARG memtrace', options.memtrace +print 'ARG regtrace', options.regtrace +print 'ARG cctrace', options.cctrace +print 'ARG printstats', options.printstats +print 'ARG verbose', options.verbose +print '' + +seed = int(options.seed) +numthreads = int(options.numthreads) +intfreq = int(options.intfreq) +zassert(intfreq > 0, 'Interrupt frequency must be greater than 0') +intrand = int(options.intrand) +progfile = options.progfile +zassert(progfile != '', 'Program file must be specified') +argv = options.argv.split(',') +zassert(len(argv) == numthreads or len(argv) == 1, 'argv: must be one per-thread or just one set of values for all threads') +procsched = options.procsched + +loadaddr = options.loadaddr +memsize = options.memsize + +memtrace = [] +if options.memtrace != '': + for m in options.memtrace.split(','): + memtrace.append(m) + +regtrace = [] +if options.regtrace != '': + for r in options.regtrace.split(','): + regtrace.append(r) + +cctrace = options.cctrace + +printstats = options.printstats +verbose = options.verbose +hdrcount = options.headercount + +# +# MAIN program +# +debug = False +debug = False + +cpu = cpu(memsize, memtrace, regtrace, cctrace, options.solve, verbose, printstats, hdrcount) + +# load a program +cpu.load(progfile, loadaddr) + +# process list +procs = proclist() +pid = 0 +stack = memsize * 1000 +for t in range(numthreads): + if len(argv) > 1: + arg = argv[pid] + else: + arg = argv[0] + procs.addproc(process(cpu, pid, loadaddr, stack, arg)) + stack -= 1000 + pid += 1 + +# get first process ready to run +procs.finalize(procsched) + +# run it +t1 = time.clock() +ic = cpu.run(procs, intfreq, intrand) +t2 = time.clock() + +if printstats: + print '' + print 'STATS:: Instructions %d' % ic + print 'STATS:: Emulation Rate %.2f kinst/sec' % (float(ic) / float(t2 - t1) / 1000.0) + +# use this for profiling +# import cProfile +# cProfile.run('run()') + + + + diff --git a/related_info/ostep/ostep12-threadlock/yield.s b/related_info/ostep/ostep12-threadlock/yield.s new file mode 100644 index 0000000..638e130 --- /dev/null +++ b/related_info/ostep/ostep12-threadlock/yield.s @@ -0,0 +1,29 @@ +.var mutex +.var count + +.main +.top + +.acquire +mov $1, %ax +xchg %ax, mutex # atomic swap of 1 and mutex +test $0, %ax # if we get 0 back: lock is free! +je .acquire_done +yield # if not, yield and try again +j .acquire +.acquire_done + +# critical section +mov count, %ax # get the value at the address +add $1, %ax # increment it +mov %ax, count # store it back + +# release lock +mov $0, mutex + +# see if we're still looping +sub $1, %bx +test $0, %bx +jgt .top + +halt diff --git a/related_info/ostep/ostep13-vsfs.md b/related_info/ostep/ostep13-vsfs.md new file mode 100644 index 0000000..4264bc4 --- /dev/null +++ b/related_info/ostep/ostep13-vsfs.md @@ -0,0 +1,256 @@ + +Use this tool, vsfs.py, to study how file system state changes as various +operations take place. The file system begins in an empty state, with just a +root directory. As the simulation takes place, various operations are +performed, thus slowly changing the on-disk state of the file system. + +The possible operations are: + +- mkdir() - creates a new directory +- creat() - creates a new (empty) file +- open(), write(), close() - appends a block to a file +- link() - creates a hard link to a file +- unlink() - unlinks a file (removing it if linkcnt==0) + +To understand how this homework functions, you must first understand how the +on-disk state of this file system is represented. The state of the file +system is shown by printing the contents of four different data structures: + +inode bitmap - indicates which inodes are allocated +inodes - table of inodes and their contents +data bitmap - indicates which data blocks are allocated +data - indicates contents of data blocks + +The bitmaps should be fairly straightforward to understand, with a 1 +indicating that the corresponding inode or data block is allocated, and a 0 +indicating said inode or data block is free. + +The inodes each have three fields: the first field indicates the type of file +(e.g., f for a regular file, d for a directory); the second indicates which +data block belongs to a file (here, files can only be empty, which would have +the address of the data block set to -1, or one block in size, which would +have a non-negative address); the third shows the reference count for the file +or directory. For example, the following inode is a regular file, which is +empty (address field set to -1), and has just one link in the file system: + + [f a:-1 r:1] + +If the same file had a block allocated to it (say block 10), it would be shown +as follows: + + [f a:10 r:1] + +If someone then created a hard link to this inode, it would then become: + + [f a:10 r:2] + +Finally, data blocks can either retain user data or directory data. If filled +with directory data, each entry within the block is of the form (name, +inumber), where "name" is the name of the file or directory, and "inumber" is +the inode number of the file. Thus, an empty root directory looks like this, +assuming the root inode is 0: + + [(.,0) (..,0)] + +If we add a single file "f" to the root directory, which has been allocated +inode number 1, the root directory contents would then become: + + [(.,0) (..,0) (f,1)] + +If a data block contains user data, it is shown as just a single character +within the block, e.g., "h". If it is empty and unallocated, just a pair of +empty brackets ([]) are shown. + +An entire file system is thus depicted as follows: + +inode bitmap 11110000 +inodes [d a:0 r:6] [f a:1 r:1] [f a:-1 r:1] [d a:2 r:2] [] ... +data bitmap 11100000 +data [(.,0) (..,0) (y,1) (z,2) (f,3)] [u] [(.,3) (..,0)] [] ... + +This file system has eight inodes and eight data blocks. The root directory +contains three entries (other than "." and ".."), to "y", "z", and "f". By +looking up inode 1, we can see that "y" is a regular file (type f), with a +single data block allocated to it (address 1). In that data block 1 are the +contents of the file "y": namely, "u". We can also see that "z" is an empty +regular file (address field set to -1), and that "f" (inode number 3) is a +directory, also empty. You can also see from the bitmaps that the first four +inode bitmap entries are marked as allocated, as well as the first three data +bitmap entries. + +The simulator can be run with the following flags: + +prompt> vsfs.py -h +Usage: vsfs.py [options] + +Options: + -h, --help show this help message and exit + -s SEED, --seed=SEED the random seed + -i NUMINODES, --numInodes=NUMINODES + number of inodes in file system + -d NUMDATA, --numData=NUMDATA + number of data blocks in file system + -n NUMREQUESTS, --numRequests=NUMREQUESTS + number of requests to simulate + -r, --reverse instead of printing state, print ops + -p, --printFinal print the final set of files/dirs + -c, --compute compute answers for me +] + +A typical usage would simply specify a random seed (to generate a different +problem), and the number of requests to simulate. In this default mode, the +simulator prints out the state of the file system at each step, and asks you +which operation must have taken place to take the file system from one state +to another. For example: + +prompt> vsfs.py -n 6 -s 16 +... +Initial state + +inode bitmap 10000000 +inodes [d a:0 r:2] [] [] [] [] [] [] [] +data bitmap 10000000 +data [(.,0) (..,0)] [] [] [] [] [] [] [] + +Which operation took place? + +inode bitmap 11000000 +inodes [d a:0 r:3] [f a:-1 r:1] [] [] [] [] [] [] +data bitmap 10000000 +data [(.,0) (..,0) (y,1)] [] [] [] [] [] [] [] + +Which operation took place? + +inode bitmap 11000000 +inodes [d a:0 r:3] [f a:1 r:1] [] [] [] [] [] [] +data bitmap 11000000 +data [(.,0) (..,0) (y,1)] [u] [] [] [] [] [] [] + +Which operation took place? + +inode bitmap 11000000 +inodes [d a:0 r:4] [f a:1 r:2] [] [] [] [] [] [] +data bitmap 11000000 +data [(.,0) (..,0) (y,1) (m,1)] [u] [] [] [] [] [] [] + +Which operation took place? + +inode bitmap 11000000 +inodes [d a:0 r:4] [f a:1 r:1] [] [] [] [] [] [] +data bitmap 11000000 +data [(.,0) (..,0) (y,1)] [u] [] [] [] [] [] [] + +Which operation took place? + +inode bitmap 11100000 +inodes [d a:0 r:5] [f a:1 r:1] [f a:-1 r:1] [] [] [] [] [] +data bitmap 11000000 +data [(.,0) (..,0) (y,1) (z,2)] [u] [] [] [] [] [] [] + +Which operation took place? + +inode bitmap 11110000 +inodes [d a:0 r:6] [f a:1 r:1] [f a:-1 r:1] [d a:2 r:2] [] ... +data bitmap 11100000 +data [(.,0) (..,0) (y,1) (z,2) (f,3)] [u] [(.,3) (..,0)] [] ... +] + +When run in this mode, the simulator just shows a series of states, and asks +what operations caused these transitions to occur. Running with the "-c" flag +shows us the answers. Specifically, file "/y" was created, a single block +appended to it, a hard link from "/m" to "/y" created, "/m" removed via a call +to unlink, the file "/z" created, and the directory "/f" created: + +prompt> vsfs.py -n 6 -s 16 -c +... +Initial state + +inode bitmap 10000000 +inodes [d a:0 r:2] [] [] [] [] [] [] [] +data bitmap 10000000 +data [(.,0) (..,0)] [] [] [] [] [] [] [] + +creat("/y"); + +inode bitmap 11000000 +inodes [d a:0 r:3] [f a:-1 r:1] [] [] [] [] [] [] +data bitmap 10000000 +data [(.,0) (..,0) (y,1)] [] [] [] [] [] [] [] + +fd=open("/y", O_WRONLY|O_APPEND); write(fd, buf, BLOCKSIZE); close(fd); + +inode bitmap 11000000 +inodes [d a:0 r:3] [f a:1 r:1] [] [] [] [] [] [] +data bitmap 11000000 +data [(.,0) (..,0) (y,1)] [u] [] [] [] [] [] [] + +link("/y", "/m"); + +inode bitmap 11000000 +inodes [d a:0 r:4] [f a:1 r:2] [] [] [] [] [] [] +data bitmap 11000000 +data [(.,0) (..,0) (y,1) (m,1)] [u] [] [] [] [] [] [] + +unlink("/m") + +inode bitmap 11000000 +inodes [d a:0 r:4] [f a:1 r:1] [] [] [] [] [] [] +data bitmap 11000000 +data [(.,0) (..,0) (y,1)] [u] [] [] [] [] [] [] + +creat("/z"); + +inode bitmap 11100000 +inodes [d a:0 r:5] [f a:1 r:1] [f a:-1 r:1] [] [] [] [] [] +data bitmap 11000000 +data [(.,0) (..,0) (y,1) (z,2)] [u] [] [] [] [] [] [] + +mkdir("/f"); + +inode bitmap 11110000 +inodes [d a:0 r:6] [f a:1 r:1] [f a:-1 r:1] [d a:2 r:2] [] ... +data bitmap 11100000 +data [(.,0) (..,0) (y,1) (z,2) (f,3)] [u] [(.,3) (..,0)] [] ... +] + +You can also run the simulator in "reverse" mode (with the "-r" flag), +printing the operations instead of the states to see if you can predict the +state changes from the given operations: + +prompt> ./vsfs.py -n 6 -s 16 -r +Initial state + +inode bitmap 10000000 +inodes [d a:0 r:2] [] [] [] [] [] [] [] +data bitmap 10000000 +data [(.,0) (..,0)] [] [] [] [] [] [] [] + +creat("/y"); + + State of file system (inode bitmap, inodes, data bitmap, data)? + +fd=open("/y", O_WRONLY|O_APPEND); write(fd, buf, BLOCKSIZE); close(fd); + + State of file system (inode bitmap, inodes, data bitmap, data)? + +link("/y", "/m"); + + State of file system (inode bitmap, inodes, data bitmap, data)? + +unlink("/m") + + State of file system (inode bitmap, inodes, data bitmap, data)? + +creat("/z"); + + State of file system (inode bitmap, inodes, data bitmap, data)? + +mkdir("/f"); + + State of file system (inode bitmap, inodes, data bitmap, data)? +] + +A few other flags control various aspects of the simulation, including the +number of inodes ("-i"), the number of data blocks ("-d"), and whether to +print the final list of all directories and files in the file system ("-p"). + diff --git a/related_info/ostep/ostep13-vsfs.py b/related_info/ostep/ostep13-vsfs.py new file mode 100755 index 0000000..3dadeb1 --- /dev/null +++ b/related_info/ostep/ostep13-vsfs.py @@ -0,0 +1,551 @@ +#! /usr/bin/env python + +import random +from optparse import OptionParser + +DEBUG = False + +def dprint(str): + if DEBUG: + print str + +printOps = True +printState = True +printFinal = True + +class bitmap: + def __init__(self, size): + self.size = size + self.bmap = [] + for num in range(size): + self.bmap.append(0) + + def alloc(self): + for num in range(len(self.bmap)): + if self.bmap[num] == 0: + self.bmap[num] = 1 + return num + return -1 + + def free(self, num): + assert(self.bmap[num] == 1) + self.bmap[num] = 0 + + def markAllocated(self, num): + assert(self.bmap[num] == 0) + self.bmap[num] = 1 + + def dump(self): + s = '' + for i in range(len(self.bmap)): + s += str(self.bmap[i]) + return s + +class block: + def __init__(self, ftype): + assert(ftype == 'd' or ftype == 'f' or ftype == 'free') + self.ftype = ftype + # only for directories, properly a subclass but who cares + self.dirUsed = 0 + self.maxUsed = 32 + self.dirList = [] + self.data = '' + + def dump(self): + if self.ftype == 'free': + return '[]' + elif self.ftype == 'd': + rc = '' + for d in self.dirList: + # d is of the form ('name', inum) + short = '(%s,%s)' % (d[0], d[1]) + if rc == '': + rc = short + else: + rc += ' ' + short + return '['+rc+']' + # return '%s' % self.dirList + else: + return '[%s]' % self.data + + def setType(self, ftype): + assert(self.ftype == 'free') + self.ftype = ftype + + def addData(self, data): + assert(self.ftype == 'f') + self.data = data + + def getNumEntries(self): + assert(self.ftype == 'd') + return self.dirUsed + + def getFreeEntries(self): + assert(self.ftype == 'd') + return self.maxUsed - self.dirUsed + + def getEntry(self, num): + assert(self.ftype == 'd') + assert(num < self.dirUsed) + return self.dirList[num] + + def addDirEntry(self, name, inum): + assert(self.ftype == 'd') + self.dirList.append((name, inum)) + self.dirUsed += 1 + assert(self.dirUsed <= self.maxUsed) + + def delDirEntry(self, name): + assert(self.ftype == 'd') + tname = name.split('/') + dname = tname[len(tname) - 1] + for i in range(len(self.dirList)): + if self.dirList[i][0] == dname: + self.dirList.pop(i) + self.dirUsed -= 1 + return + assert(1 == 0) + + def dirEntryExists(self, name): + assert(self.ftype == 'd') + for d in self.dirList: + if name == d[0]: + return True + return False + + def free(self): + assert(self.ftype != 'free') + if self.ftype == 'd': + # check for only dot, dotdot here + assert(self.dirUsed == 2) + self.dirUsed = 0 + self.data = '' + self.ftype = 'free' + +class inode: + def __init__(self, ftype='free', addr=-1, refCnt=1): + self.setAll(ftype, addr, refCnt) + + def setAll(self, ftype, addr, refCnt): + assert(ftype == 'd' or ftype == 'f' or ftype == 'free') + self.ftype = ftype + self.addr = addr + self.refCnt = refCnt + + def incRefCnt(self): + self.refCnt += 1 + + def decRefCnt(self): + self.refCnt -= 1 + + def getRefCnt(self): + return self.refCnt + + def setType(self, ftype): + assert(ftype == 'd' or ftype == 'f' or ftype == 'free') + self.ftype = ftype + + def setAddr(self, block): + self.addr = block + + def getSize(self): + if self.addr == -1: + return 0 + else: + return 1 + + def getAddr(self): + return self.addr + + def getType(self): + return self.ftype + + def free(self): + self.ftype = 'free' + self.addr = -1 + + +class fs: + def __init__(self, numInodes, numData): + self.numInodes = numInodes + self.numData = numData + + self.ibitmap = bitmap(self.numInodes) + self.inodes = [] + for i in range(self.numInodes): + self.inodes.append(inode()) + + self.dbitmap = bitmap(self.numData) + self.data = [] + for i in range(self.numData): + self.data.append(block('free')) + + # root inode + self.ROOT = 0 + + # create root directory + self.ibitmap.markAllocated(self.ROOT) + self.inodes[self.ROOT].setAll('d', 0, 2) + self.dbitmap.markAllocated(self.ROOT) + self.data[0].setType('d') + self.data[0].addDirEntry('.', self.ROOT) + self.data[0].addDirEntry('..', self.ROOT) + + # these is just for the fake workload generator + self.files = [] + self.dirs = ['/'] + self.nameToInum = {'/':self.ROOT} + + def dump(self): + print 'inode bitmap ', self.ibitmap.dump() + print 'inodes ', + for i in range(0,self.numInodes): + ftype = self.inodes[i].getType() + if ftype == 'free': + print '[]', + else: + print '[%s a:%s r:%d]' % (ftype, self.inodes[i].getAddr(), self.inodes[i].getRefCnt()), + print '' + print 'data bitmap ', self.dbitmap.dump() + print 'data ', + for i in range(self.numData): + print self.data[i].dump(), + print '' + + def makeName(self): + p = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'j', 'k', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'] + return p[int(random.random() * len(p))] + p = ['b', 'c', 'd', 'f', 'g', 'h', 'j', 'k', 'l', 'm', 'n', 'p', 's', 't', 'v', 'w', 'x', 'y', 'z'] + f = p[int(random.random() * len(p))] + p = ['a', 'e', 'i', 'o', 'u'] + s = p[int(random.random() * len(p))] + p = ['b', 'c', 'd', 'f', 'g', 'j', 'k', 'l', 'm', 'n', 'p', 's', 't', 'v', 'w', 'x', 'y', 'z'] + l = p[int(random.random() * len(p))] + return '%c%c%c' % (f, s, l) + + def inodeAlloc(self): + return self.ibitmap.alloc() + + def inodeFree(self, inum): + self.ibitmap.free(inum) + self.inodes[inum].free() + + def dataAlloc(self): + return self.dbitmap.alloc() + + def dataFree(self, bnum): + self.dbitmap.free(bnum) + self.data[bnum].free() + + def getParent(self, name): + tmp = name.split('/') + if len(tmp) == 2: + return '/' + pname = '' + for i in range(1, len(tmp)-1): + pname = pname + '/' + tmp[i] + return pname + + def deleteFile(self, tfile): + if printOps: + print 'unlink("%s");' % tfile + + inum = self.nameToInum[tfile] + + if self.inodes[inum].getRefCnt() == 1: + # free data blocks first + dblock = self.inodes[inum].getAddr() + if dblock != -1: + self.dataFree(dblock) + # then free inode + self.inodeFree(inum) + else: + self.inodes[inum].decRefCnt() + + # remove from parent directory + parent = self.getParent(tfile) + # print '--> delete from parent', parent + pinum = self.nameToInum[parent] + # print '--> delete from parent inum', pinum + pblock = self.inodes[pinum].getAddr() + # FIXED BUG: DECREASE PARENT INODE REF COUNT! (thanks to Srinivasan Thirunarayanan) + self.inodes[pinum].decRefCnt() + # print '--> delete from parent addr', pblock + self.data[pblock].delDirEntry(tfile) + + # finally, remove from files list + self.files.remove(tfile) + return 0 + + def createLink(self, target, newfile, parent): + # find info about parent + parentInum = self.nameToInum[parent] + + # is there room in the parent directory? + pblock = self.inodes[parentInum].getAddr() + if self.data[pblock].getFreeEntries() <= 0: + dprint('*** createLink failed: no room in parent directory ***') + return -1 + + # print 'is %s in directory %d' % (newfile, pblock) + if self.data[pblock].dirEntryExists(newfile): + dprint('*** createLink failed: not a unique name ***') + return -1 + + # now, find inumber of target + tinum = self.nameToInum[target] + self.inodes[tinum].incRefCnt() + + # inc parent ref count + self.inodes[parentInum].incRefCnt() + + # now add to directory + tmp = newfile.split('/') + ename = tmp[len(tmp)-1] + self.data[pblock].addDirEntry(ename, tinum) + return tinum + + def createFile(self, parent, newfile, ftype): + # find info about parent + parentInum = self.nameToInum[parent] + + # is there room in the parent directory? + pblock = self.inodes[parentInum].getAddr() + if self.data[pblock].getFreeEntries() <= 0: + dprint('*** createFile failed: no room in parent directory ***') + return -1 + + # have to make sure file name is unique + block = self.inodes[parentInum].getAddr() + # print 'is %s in directory %d' % (newfile, block) + if self.data[block].dirEntryExists(newfile): + dprint('*** createFile failed: not a unique name ***') + return -1 + + # find free inode + inum = self.inodeAlloc() + if inum == -1: + dprint('*** createFile failed: no inodes left ***') + return -1 + + # if a directory, have to allocate directory block for basic (., ..) info + fblock = -1 + if ftype == 'd': + refCnt = 2 + fblock = self.dataAlloc() + if fblock == -1: + dprint('*** createFile failed: no data blocks left ***') + self.inodeFree(inum) + return -1 + else: + self.data[fblock].setType('d') + self.data[fblock].addDirEntry('.', inum) + self.data[fblock].addDirEntry('..', parentInum) + else: + refCnt = 1 + + # now ok to init inode properly + self.inodes[inum].setAll(ftype, fblock, refCnt) + + # inc parent ref count + self.inodes[parentInum].incRefCnt() + + # and add to directory of parent + self.data[pblock].addDirEntry(newfile, inum) + return inum + + def writeFile(self, tfile, data): + inum = self.nameToInum[tfile] + curSize = self.inodes[inum].getSize() + dprint('writeFile: inum:%d cursize:%d refcnt:%d' % (inum, curSize, self.inodes[inum].getRefCnt())) + if curSize == 1: + dprint('*** writeFile failed: file is full ***') + return -1 + fblock = self.dataAlloc() + if fblock == -1: + dprint('*** writeFile failed: no data blocks left ***') + return -1 + else: + self.data[fblock].setType('f') + self.data[fblock].addData(data) + self.inodes[inum].setAddr(fblock) + if printOps: + print 'fd=open("%s", O_WRONLY|O_APPEND); write(fd, buf, BLOCKSIZE); close(fd);' % tfile + return 0 + + def doDelete(self): + dprint('doDelete') + if len(self.files) == 0: + return -1 + dfile = self.files[int(random.random() * len(self.files))] + dprint('try delete(%s)' % dfile) + return self.deleteFile(dfile) + + def doLink(self): + dprint('doLink') + if len(self.files) == 0: + return -1 + parent = self.dirs[int(random.random() * len(self.dirs))] + nfile = self.makeName() + + # pick random target + target = self.files[int(random.random() * len(self.files))] + + # get full name of newfile + if parent == '/': + fullName = parent + nfile + else: + fullName = parent + '/' + nfile + + dprint('try createLink(%s %s %s)' % (target, nfile, parent)) + inum = self.createLink(target, nfile, parent) + if inum >= 0: + self.files.append(fullName) + self.nameToInum[fullName] = inum + if printOps: + print 'link("%s", "%s");' % (target, fullName) + return 0 + return -1 + + def doCreate(self, ftype): + dprint('doCreate') + parent = self.dirs[int(random.random() * len(self.dirs))] + nfile = self.makeName() + if ftype == 'd': + tlist = self.dirs + else: + tlist = self.files + + if parent == '/': + fullName = parent + nfile + else: + fullName = parent + '/' + nfile + + dprint('try createFile(%s %s %s)' % (parent, nfile, ftype)) + inum = self.createFile(parent, nfile, ftype) + if inum >= 0: + tlist.append(fullName) + self.nameToInum[fullName] = inum + if parent == '/': + parent = '' + if ftype == 'd': + if printOps: + print 'mkdir("%s/%s");' % (parent, nfile) + else: + if printOps: + print 'creat("%s/%s");' % (parent, nfile) + return 0 + return -1 + + def doAppend(self): + dprint('doAppend') + if len(self.files) == 0: + return -1 + afile = self.files[int(random.random() * len(self.files))] + dprint('try writeFile(%s)' % afile) + data = chr(ord('a') + int(random.random() * 26)) + rc = self.writeFile(afile, data) + return rc + + def run(self, numRequests): + self.percentMkdir = 0.40 + self.percentWrite = 0.40 + self.percentDelete = 0.20 + self.numRequests = 20 + + print 'Initial state' + print '' + self.dump() + print '' + + for i in range(numRequests): + if printOps == False: + print 'Which operation took place?' + rc = -1 + while rc == -1: + r = random.random() + if r < 0.3: + rc = self.doAppend() + dprint('doAppend rc:%d' % rc) + elif r < 0.5: + rc = self.doDelete() + dprint('doDelete rc:%d' % rc) + elif r < 0.7: + rc = self.doLink() + dprint('doLink rc:%d' % rc) + else: + if random.random() < 0.75: + rc = self.doCreate('f') + dprint('doCreate(f) rc:%d' % rc) + else: + rc = self.doCreate('d') + dprint('doCreate(d) rc:%d' % rc) + if printState == True: + print '' + self.dump() + print '' + else: + print '' + print ' State of file system (inode bitmap, inodes, data bitmap, data)?' + print '' + + if printFinal: + print '' + print 'Summary of files, directories::' + print '' + print ' Files: ', self.files + print ' Directories:', self.dirs + print '' + +# +# main program +# +parser = OptionParser() + +parser.add_option('-s', '--seed', default=0, help='the random seed', action='store', type='int', dest='seed') +parser.add_option('-i', '--numInodes', default=8, help='number of inodes in file system', action='store', type='int', dest='numInodes') +parser.add_option('-d', '--numData', default=8, help='number of data blocks in file system', action='store', type='int', dest='numData') +parser.add_option('-n', '--numRequests', default=10, help='number of requests to simulate', action='store', type='int', dest='numRequests') +parser.add_option('-r', '--reverse', default=False, help='instead of printing state, print ops', action='store_true', dest='reverse') +parser.add_option('-p', '--printFinal', default=False, help='print the final set of files/dirs', action='store_true', dest='printFinal') +parser.add_option('-c', '--compute', default=False, help='compute answers for me', action='store_true', dest='solve') + +(options, args) = parser.parse_args() + +print 'ARG seed', options.seed +print 'ARG numInodes', options.numInodes +print 'ARG numData', options.numData +print 'ARG numRequests', options.numRequests +print 'ARG reverse', options.reverse +print 'ARG printFinal', options.printFinal +print '' + +random.seed(options.seed) + +if options.reverse: + printState = False + printOps = True +else: + printState = True + printOps = False + +if options.solve: + printOps = True + printState = True + +printFinal = options.printFinal + +# +# have to generate RANDOM requests to the file system +# that are VALID! +# + +f = fs(options.numInodes, options.numData) + +# +# ops: mkdir rmdir : create delete : append write +# + +f.run(options.numRequests) + + diff --git a/related_info/ostep/ostep14-afs.md b/related_info/ostep/ostep14-afs.md new file mode 100644 index 0000000..fcfb8bc --- /dev/null +++ b/related_info/ostep/ostep14-afs.md @@ -0,0 +1,252 @@ + +This program, afs.py, allows you to experiment with the cache consistency +behavior of the Andrew File System (AFS). The program generates random client +traces (of file opens, reads, writes, and closes), enabling the user to see if +they can predict what values end up in various files. + +Here is an example run: + +prompt> ./afs.py -C 2 -n 1 -s 12 + + Server c0 c1 +file:a contains:0 + open:a [fd:0] + write:0 value? -> 1 + close:0 + open:a [fd:0] + read:0 -> value? + close:0 +file:a contains:? +prompt> + +The trace is fairly simple to read. On the left is the server, and each column +shows actions being taken on each of two clients (use -C to specify +a different number). Each client generates one random action (-n 1), which is +either the open/read/close of a file or the open/write/close of a file. The +contents of a file, for simplicity, is always just a single number. + +To generate different traces, use '-s' (for a random seed), as always. Here we +set it to 12 to get this specific trace. + +In the trace, the server shows the initial contents of all the files in the +system: + +file:a contains:0 + +As you can see in this trace, there is just one file (a) and it contains the +value 0. + +Time increases downwards, and what is next is client 0 (c0) opening the file +'a' (which returns a file descriptor, 0 in this case), writing to that +descriptor, and then closing the file. + +Immediately you see the first question posed to you: + + write:0 value? -> 1 + +When writing to descriptor 0, you are overwriting an existing value with the +new value of 1. What was that old value? (pretty easy in this case: 0). + +Then client 1 begins doing some work (c1). It opens the file, reads it, and +closes it. Again, we have a question to answer: + + read:0 -> value? + +When reading from this file, what value should client 1 see? Again, given AFS +consistency, the answer is straightforward: 1 (the value placed in the file +when c0 closed the file and updated the server). + +The final question in the trace is the final value of the file on the server: + +file:a contains:? + +Again, the answer here is easy: 1 (as generated by c0). + +To see if you have answered these questions correctly, run with the '-c' flag +(or '--compute'), as follows: + +prompt> ./afs.py -C 2 -n 1 -s 12 -c + + Server c0 c1 +file:a contains:0 + open:a [fd:0] + write:0 0 -> 1 + close:0 + open:a [fd:0] + read:0 -> 1 + close:0 +file:a contains:1 +prompt> + +From this trace, you can see that all the question marks have been filled in +with answers. + +More detail is available on what has happened too, with the '-d' ('--detail') +flag. Here is an example that shows when each client issued a get or put of a +file to the server: + +prompt> ./afs.py -C 2 -n 1 -s 12 -c -d 1 + + Server c0 c1 +file:a contains:0 + open:a [fd:0] +getfile:a c:c0 [0] + + write:0 0 -> 1 + + close:0 +putfile:a c:c0 [1] + + open:a [fd:0] +getfile:a c:c1 [1] + + read:0 -> 1 + + close:0 + +file:a contains:1 +prompt> + +You can show more with higher levels of detail, including cache invalidations, +the exact client cache state after each step, and extra diagnostic +information. We'll show these in one more example below. + +Random client actions are useful to generate new problems and try to solve +them; however, in some cases it is useful to control exactly what each client +does in order to see specific AFS behaviors. To do this, you can use the '-A' +and '-S' flags (either together or in tandem). + +The '-S' flag lets you control the exact schedule of client actions. Assume our +example above. Let's say we wish to run client 1 in entirety first; to achieve +this end, we simply run the following: + +prompt> ./afs.py -C 2 -n 1 -s 12 -S 111000 + + Server c0 c1 +file:a contains:0 + open:a [fd:0] + read:0 -> value? + close:0 + open:a [fd:0] + write:0 value? -> 1 + close:0 +file:a contains:? +prompt> + +The -S flag here is passed "111000" which means "run client 1, then client 1, +then 1 again, then 0, 0, 0, and then repeat (if need be)". The result in this +case is client 1 reading file a before client 1 writes it. + +The '-A' flag gives exact control over which actions the clients take. Here is +an example: + +prompt> ./afs.py -s 12 -S 011100 -A oa1:r1:c1,oa1:w1:c1 + + Server c0 c1 +file:a contains:0 + open:a [fd:1] + open:a [fd:1] + write:1 value? -> 1 + close:1 + read:1 -> value? + close:1 +file:a contains:? +prompt> + +In this example, we have specified the following via -A "oa1:r1:c1,oa1:w1:c1". +The list splits each clients actions by a comma; thus, client 0 should do +whatever "oa1:r1:c1" indicates, whereas client 1 should do whatever the string +"oa1:w1:c1" indicates. Parsing each command string is straightforward: "oa1" +means open file 'a' and assign it file descriptor 1; "r1" or "w1" means read +or write file descriptor 1; "c1" means close file descriptor 1. + +So what value will the read on client 0 return? + +We can also see the cache state, callbacks, and invalidations with a few extra +flags (-d 7): + +prompt> ./afs.py -s 12 -S 011100 -A oa1:r1:c1,oa1:w1:c1 -c -d 7 + + Server c0 c1 +file:a contains:0 + open:a [fd:1] +getfile:a c:c0 [0] + [a: 0 (v=1,d=0,r=1)] + + open:a [fd:1] +getfile:a c:c1 [0] + [a: 0 (v=1,d=0,r=1)] + + write:1 0 -> 1 + [a: 1 (v=1,d=1,r=1)] + + close:1 +putfile:a c:c1 [1] +callback: c:c0 file:a + invalidate a + [a: 0 (v=0,d=0,r=1)] + [a: 1 (v=1,d=0,r=0)] + + read:1 -> 0 + [a: 0 (v=0,d=0,r=1)] + + close:1 + +file:a contains:1 +prompt> + +From this trace, we can see what happens when client 1 closes the (modified) +file. At that point, c1 puts the file to the server. The server knows that c0 +has the file cached, and thus sends an invalidation to c0. However, c0 already +has the file open; as a result, the cache keeps the old contents until the +file is closed. + +You can see this in tracking the cache contents throughout the trace +(available with the correct -d flag, in particular any value which sets the +3rd least significant bit to 1, such as -d 4, -d 5, -d 6, -d 7, etc.). When +client 0 opens the file, you see the following cache state after the open is +finished: + + [a: 0 (v=1,d=0,r=1)] + +This means file 'a' is in the cache with value '0', and has three bits of +state associated with it: v (valid), d (dirty), and r (reference count). The +valid bit tracks whether the contents are valid; it is now, because the cache +has not been invalidated by a callback (yet). The dirty bit changes when the +file has been written to and must be flushed back to the server when +closed. Finally, the reference count tracks how many times the file has been +opened (but not yet closed); this is used to ensure the client gets the old +value of the file until it's been closed by all readers and then re-opened. + +The full list of options is available here: + +Options: + -h, --help show this help message and exit + -s SEED, --seed=SEED the random seed + -C NUMCLIENTS, --clients=NUMCLIENTS + number of clients + -n NUMSTEPS, --numsteps=NUMSTEPS + ops each client will do + -f NUMFILES, --numfiles=NUMFILES + number of files in server + -r READRATIO, --readratio=READRATIO + ratio of reads/writes + -A ACTIONS, --actions=ACTIONS + client actions exactly specified, e.g., + oa1:r1:c1,oa1:w1:c1 specifies two clients; each opens + the file a, client 0 reads it whereas client 1 writes + it, and then each closes it + -S SCHEDULE, --schedule=SCHEDULE + exact schedule to run; 01 alternates round robin + between clients 0 and 1. Left unspecified leads to + random scheduling + -p, --printstats print extra stats + -c, --compute compute answers for me + -d DETAIL, --detail=DETAIL + detail level when giving answers (1:server + actions,2:invalidations,4:client cache,8:extra + labels); OR together for multiple + +Read the AFS chapter, and answer the questions at the back, or just explore +this simulator more on your own to increase your understanding of AFS. + diff --git a/related_info/ostep/ostep14-afs.py b/related_info/ostep/ostep14-afs.py new file mode 100644 index 0000000..494ec04 --- /dev/null +++ b/related_info/ostep/ostep14-afs.py @@ -0,0 +1,600 @@ +#! /usr/bin/env python + +import random +from optparse import OptionParser +import string + +def tprint(str): + print str + +def dprint(str): + return + +def dospace(howmuch): + for i in range(howmuch + 1): + print '%28s' % ' ', + +# given list, pick random element and return it +def pickrand(tlist): + n = int(random.random() * len(tlist)) + p = tlist[n] + return p + +# given number, conclude if nth bit is set +def isset(num, index): + mask = 1 << index + return (num & mask) > 0 + +# useful instead of assert +def zassert(cond, str): + if cond == False: + print 'ABORT::', str + exit(1) + +# +# Which files are used in the simulation +# +# Not representing a realistic piece of anything +# but rather just for convenience when generating +# random traces ... +# +# Files are named 'a', 'b', etc. for ease of use +# Could probably add a numeric aspect to allow +# for more than 26 files but who cares +# + +class files: + def __init__(self, numfiles): + self.numfiles = numfiles + self.value = 0 + self.filelist = list(string.ascii_lowercase)[0:numfiles] + + def getfiles(self): + return self.filelist + + def getvalue(self): + rc = self.value + self.value += 1 + return rc + +# +# Models the actions of the AFS server +# +# The only real interactions are get/put +# get() causes the server to track which files cache what; +# put() may cause callbacks to invalidate client caches +# +class server: + def __init__(self, files, solve, detail): + self.files = files + self.solve = solve + self.detail = detail + + flist = self.files.getfiles() + self.contents = {} + for f in flist: + v = self.files.getvalue() + self.contents[f] = v + self.getcnt, self.putcnt = 0, 0 + + def stats(self): + print 'Server -- Gets:%d Puts:%d' % (self.getcnt, self.putcnt) + + def filestats(self, printcontents): + for fname in self.contents: + if printcontents: + print('file:%s contains:%d' % (fname, self.contents[fname])) + else: + print('file:%s contains:?' % fname) + + + def setclients(self, clients): + # need list of clients + self.clients = clients + + # per client callback list + self.cache = {} + for c in self.clients: + self.cache[c.getname()] = [] + + def get(self, client, fname): + zassert(fname in self.contents, 'server:get() -- file:%s not found on server' % fname) + self.getcnt += 1 + if self.solve and isset(self.detail, 0): + print('getfile:%s c:%s [%d]' % (fname, client, self.contents[fname])) + if fname not in self.cache[client]: + self.cache[client].append(fname) + # dprint(' -> List for client %s' % client, ' is ', self.cache[client]) + return self.contents[fname] + + def put(self, client, fname, value): + zassert(fname in self.contents, 'server:put() -- file:%s not found on server' % fname) + self.putcnt += 1 + self.contents[fname] = value + if self.solve and isset(self.detail, 0): + print('putfile:%s c:%s [%s]' % (fname, client, self.contents[fname])) + # scan others for callback + for c in self.clients: + cname = c.getname() + if fname in self.cache[cname] and cname != client: + if self.solve and isset(self.detail, 1): + print 'callback: c:%s file:%s' % (cname, fname) + c.invalidate(fname) + self.cache[cname].remove(fname) + +# +# Per-client file descriptors +# +# Would be useful if the simulation allowed more +# than one active file open() at a time; it kind +# of does but this isn't really utilized +# +class filedesc: + def __init__(self, max=1024): + self.max = max + self.fd = {} + for i in range(self.max): + self.fd[i] = '' + + def alloc(self, fname, sfd=-1): + if sfd != -1: + zassert(self.fd[sfd] == '', 'filedesc:alloc() -- fd:%d already in use, cannot allocate' % sfd) + self.fd[sfd] = fname + return sfd + else: + for i in range(self.max): + if self.fd[i] == '': + self.fd[i] = fname + return i + return -1 + + def lookup(self, sfd): + zassert(i >= 0 and i < self.max, 'filedesc:lookup() -- file descriptor out of valid range (%d not between 0 and %d)' % (sfd, self.max)) + zassert(self.fd[sfd] != '', 'filedesc:lookup() -- fd:%d not in use, cannot lookup' % sfd) + return self.fd[sfd] + + def free(self, i): + zassert(i >= 0 and i < self.max, 'filedesc:free() -- file descriptor out of valid range (%d not between 0 and %d)' % (sfd, self.max)) + zassert(self.fd[sfd] != '', 'filedesc:free() -- fd:%d not in use, cannot free' % sfd) + self.fd[i] = '' + +# +# The client cache +# +# Just models what files are cached. +# When a file is opened, its contents are fetched +# from the server and put in the cache. At that point, +# the cache contents are VALID, DIRTY/NOT (depending +# on whether this is for reading or writing), and the +# REFERENCE COUNT is set to 1. If multiple open's take +# place on this file, REFERENCE COUNT will be updated +# accordingly. VALID gets set to 0 if the cache is +# invalidated by a callback; however, the contents +# still might be used by a given client if the file +# is already open. Note that a callback does NOT +# prevent a client from overwriting an already opened file. +# +class cache: + def __init__(self, name, num, solve, detail): + self.name = name + self.num = num + self.solve = solve + self.detail = detail + + self.cache = {} + + self.hitcnt = 0 + self.misscnt = 0 + self.invalidcnt = 0 + + def stats(self): + print ' Cache -- Hits:%d Misses:%d Invalidates:%d' % (self.hitcnt, self.misscnt, self.invalidcnt) + + def put(self, fname, data, dirty, refcnt): + self.cache[fname] = dict(data=data, dirty=dirty, refcnt=refcnt, valid=True) + + def update(self, fname, data): + self.cache[fname] = dict(data=data, dirty=True, refcnt=self.cache[fname]['refcnt'], valid=self.cache[fname]['valid']) + + def invalidate(self, fname): + zassert(fname in self.cache, 'cache:invalidate() -- cannot invalidate file not in cache (%s)' % fname) + self.invalidcnt += 1 + self.cache[fname] = dict(data=self.cache[fname]['data'], dirty=self.cache[fname]['dirty'], + refcnt=self.cache[fname]['refcnt'], valid=False) + if self.solve and isset(self.detail, 1): + dospace(self.num) + if isset(self.detail,3): + print '%2s invalidate %s' % (self.name, fname) + else: + print 'invalidate %s' % (fname) + self.printstate(self.num) + + def checkvalid(self, fname): + zassert(fname in self.cache, 'cache:checkvalid() -- cannot checkvalid on file not in cache (%s)' % fname) + if self.cache[fname]['valid'] == False and self.cache[fname]['refcnt'] == 0: + del self.cache[fname] + + def printstate(self, fname): + for fname in self.cache: + data = self.cache[fname]['data'] + dirty = self.cache[fname]['dirty'] + refcnt = self.cache[fname]['refcnt'] + valid = self.cache[fname]['valid'] + if valid == True: + validPrint = 1 + else: + validPrint = 0 + if dirty == True: + dirtyPrint = 1 + else: + dirtyPrint = 0 + + if self.solve and isset(self.detail, 2): + dospace(self.num) + if isset(self.detail, 3): + print '%s [%s:%2d (v=%d,d=%d,r=%d)]' % (self.name, fname, data, validPrint, dirtyPrint, refcnt) + else: + print '[%s:%2d (v=%d,d=%d,r=%d)]' % (fname, data, validPrint, dirtyPrint, refcnt) + + def checkget(self, fname): + if fname in self.cache: + self.cache[fname] = dict(data=self.cache[fname]['data'], dirty=self.cache[fname]['dirty'], + refcnt=self.cache[fname]['refcnt'], valid=self.cache[fname]['valid']) + self.hitcnt += 1 + return (True, self.cache[fname]) + self.misscnt += 1 + return (False, -1) + + def get(self, fname): + assert(fname in self.cache) + return (True, self.cache[fname]) + + def incref(self, fname): + assert(fname in self.cache) + self.cache[fname] = dict(data=self.cache[fname]['data'], dirty=self.cache[fname]['dirty'], + refcnt=self.cache[fname]['refcnt'] + 1, valid=self.cache[fname]['valid']) + + def decref(self, fname): + assert(fname in self.cache) + self.cache[fname] = dict(data=self.cache[fname]['data'], dirty=self.cache[fname]['dirty'], + refcnt=self.cache[fname]['refcnt'] - 1, valid=self.cache[fname]['valid']) + + def setdirty(self, fname, dirty): + assert(fname in self.cache) + self.cache[fname] = dict(data=self.cache[fname]['data'], dirty=dirty, + refcnt=self.cache[fname]['refcnt'], valid=self.cache[fname]['valid']) + + def setclean(self, fname): + assert(fname in self.cache) + self.cache[fname] = dict(data=self.cache[fname]['data'], dirty=False, + refcnt=self.cache[fname]['refcnt'], valid=self.cache[fname]['valid']) + + def isdirty(self, fname): + assert(fname in self.cache) + return (self.cache[fname]['dirty'] == True) + + def setvalid(self, fname): + assert(fname in self.cache) + self.cache[fname] = dict(data=self.cache[fname]['data'], dirty=self.cache[fname]['dirty'], + refcnt=self.cache[fname]['refcnt'], valid=True) + + +# actions +MICRO_OPEN = 1 +MICRO_READ = 2 +MICRO_WRITE = 3 +MICRO_CLOSE = 4 + +def op2name(op): + if op == MICRO_OPEN: + return 'MICRO_OPEN' + elif op == MICRO_READ: + return 'MICRO_READ' + elif op == MICRO_WRITE: + return 'MICRO_WRITE' + elif op == MICRO_CLOSE: + return 'MICRO_CLOSE' + else: + abort('error: bad op -> ' + op) + +# +# Client class +# +# Models the behavior of each client in the system. +# +# +# +class client: + def __init__(self, name, cid, server, files, bias, numsteps, actions, solve, detail): + self.name = name # readable name of client + self.cid = cid # client ID + self.server = server # server object + self.files = files # files object + self.bias = bias # bias + self.actions = actions # schedule exactly? + self.solve = solve # show answers? + self.detail = detail # how much of an answer to show + + # cache + self.cache = cache(self.name, self.cid, self.solve, self.detail) + + # file desc + self.fd = filedesc() + + # stats + self.readcnt = 0 + self.writecnt = 0 + + # init actions + self.done = False # track state + self.acnt = 0 # this is used when running + self.acts = [] # this just tracks the opcodes + + if self.actions == '': + # in case with no specific actions, generate one... + for i in range(numsteps): + fname = pickrand(self.files.getfiles()) + r = random.random() + fd = self.fd.alloc(fname) + zassert(fd >= 0, 'client:init() -- ran out of file descriptors, sorry!') + if r < self.bias[0]: + # FILE_READ + self.acts.append((MICRO_OPEN, fname, fd)) + self.acts.append((MICRO_READ, fd)) + self.acts.append((MICRO_CLOSE, fd)) + else: + # FILE_WRITE + self.acts.append((MICRO_OPEN, fname, fd)) + self.acts.append((MICRO_WRITE, fd)) + self.acts.append((MICRO_CLOSE, fd)) + else: + # in this case, unpack actions and make it happen + # should look like this: "oa1:ra1:ca1" (open 'a' for reading with file desc 1, read from file a (fd:1), etc.) + # yes the file descriptor and file name are redundant for read/write and close + for a in self.actions.split(':'): + act = a[0] + if act == 'o': + zassert(len(a) == 3, 'client:init() -- malformed open action (%s) should be oa1 or something like that' % a) + fname, fd = a[1], int(a[2]) + self.fd.alloc(fname, fd) + assert(fd >= 0) + self.acts.append((MICRO_OPEN, fname, fd)) + elif act == 'r': + zassert(len(a) == 2, 'client:init() -- malformed read action (%s) should be r1 or something like that' % a) + fd = int(a[1]) + self.acts.append((MICRO_READ, fd)) + elif act == 'w': + zassert(len(a) == 2, 'client:init() -- malformed write action (%s) should be w1 or something like that' % a) + fd = int(a[1]) + self.acts.append((MICRO_WRITE, fd)) + elif act == 'c': + zassert(len(a) == 2, 'client:init() -- malformed close action (%s) should be c1 or something like that' % a) + fd = int(a[1]) + self.acts.append((MICRO_CLOSE, fd)) + else: + print 'Unrecognized command: %s (from %s)' % (act, a) + exit(1) + # debug ACTS + # print self.acts + + def getname(self): + return self.name + + def stats(self): + print '%s -- Reads:%d Writes:%d' % (self.name, self.readcnt, self.writecnt) + self.cache.stats() + + def getfile(self, fname, dirty): + (incache, item) = self.cache.checkget(fname) + if incache == True and item['valid'] == 1: + dprint(' -> CLIENT %s:: HAS LOCAL COPY of %s' % (self.name, fname)) + self.cache.setdirty(fname, dirty) + else: + data = self.server.get(self.name, fname) + self.cache.put(fname, data, dirty, 0) + self.cache.incref(fname) + + def putfile(self, fname, value): + self.server.put(self.name, fname, value) + self.cache.setclean(fname) + self.cache.setvalid(fname) + + def invalidate(self, fname): + self.cache.invalidate(fname) + + def step(self, space): + if self.done == True: + return -1 + if self.acnt == len(self.acts): + self.done = True + return 0 + + # now figure out what to do and do it + # action, fname, fd = self.acts[self.acnt] + action = self.acts[self.acnt][0] + + # print '' + # print '*************************' + # print '%s ACTION -> %s' % (self.name, op2name(action)) + # print '*************************' + + # first, do spacing for command (below) + dospace(space) + + if isset(self.detail, 3) == True: + print self.name, + + # now handle the action + if action == MICRO_OPEN: + fname, fd = self.acts[self.acnt][1], self.acts[self.acnt][2] + tprint('open:%s [fd:%d]' % (fname, fd)) + self.getfile(fname, dirty=False) + elif action == MICRO_READ: + fd = self.acts[self.acnt][1] + fname = self.fd.lookup(fd) + self.readcnt += 1 + incache, contents = self.cache.get(fname) + assert(incache == True) + if self.solve: + tprint('read:%d -> %d' % (fd, contents['data'])) + else: + tprint('read:%d -> value?' % (fd)) + elif action == MICRO_WRITE: + fd = self.acts[self.acnt][1] + fname = self.fd.lookup(fd) + self.writecnt += 1 + incache, contents = self.cache.get(fname) + assert(incache == True) + v = self.files.getvalue() + self.cache.update(fname, v) + if self.solve: + tprint('write:%d %d -> %d' % (fd, contents['data'], v)) + else: + tprint('write:%d value? -> %d' % (fd, v)) + elif action == MICRO_CLOSE: + fd = self.acts[self.acnt][1] + fname = self.fd.lookup(fd) + incache, contents = self.cache.get(fname) + assert(incache == True) + tprint('close:%d' % (fd)) + if self.cache.isdirty(fname): + self.putfile(fname, contents['data']) + self.cache.decref(fname) + self.cache.checkvalid(fname) + + # useful to see + self.cache.printstate(self.name) + + if self.solve and self.detail > 0: + print '' + + # return that there is more left to do + self.acnt += 1 + return 1 + + +# +# main program +# +parser = OptionParser() +parser.add_option('-s', '--seed', default=0, help='the random seed', action='store', type='int', dest='seed') +parser.add_option('-C', '--clients', default=2, help='number of clients', action='store', type='int', dest='numclients') +parser.add_option('-n', '--numsteps', default=2, help='ops each client will do', action='store', type='int', dest='numsteps') +parser.add_option('-f', '--numfiles', default=1, help='number of files in server', action='store', type='int', dest='numfiles') +parser.add_option('-r', '--readratio', default=0.5, help='ratio of reads/writes', action='store', type='float', dest='readratio') +parser.add_option('-A', '--actions', default='', help='client actions exactly specified, e.g., oa1:r1:c1,oa1:w1:c1 specifies two clients; each opens the file a, client 0 reads it whereas client 1 writes it, and then each closes it', + action='store', type='string', dest='actions') +parser.add_option('-S', '--schedule', default='', help='exact schedule to run; 01 alternates round robin between clients 0 and 1. Left unspecified leads to random scheduling', + action='store', type='string', dest='schedule') +parser.add_option('-p', '--printstats', default=False, help='print extra stats', action='store_true', dest='printstats') +parser.add_option('-c', '--compute', default=False, help='compute answers for me', action='store_true', dest='solve') +parser.add_option('-d', '--detail', default=0, help='detail level when giving answers (1:server actions,2:invalidations,4:client cache,8:extra labels); OR together for multiple', action='store', type='int', dest='detail') +(options, args) = parser.parse_args() + +print 'ARG seed', options.seed +print 'ARG numclients', options.numclients +print 'ARG numsteps', options.numsteps +print 'ARG numfiles', options.numfiles +print 'ARG readratio', options.readratio +print 'ARG actions', options.actions +print 'ARG schedule', options.schedule +print 'ARG detail', options.detail +print '' + +seed = int(options.seed) +numclients = int(options.numclients) +numsteps = int(options.numsteps) +numfiles = int(options.numfiles) +readratio = float(options.readratio) +actions = options.actions +schedule = options.schedule +printstats = options.printstats +solve = options.solve +detail = options.detail + +# with specific schedule, files are all specified by a single letter in specific actions list +# but we ignore this for now... + +zassert(numfiles > 0 and numfiles <= 26, 'main: can only simulate 26 or fewer files, sorry') +zassert(readratio >= 0.0 and readratio <= 1.0, 'main: read ratio must be between 0 and 1 inclusive') + +# start it +random.seed(seed) + +# files in server to begin with +f = files(numfiles) + +# make server +s = server(f, solve, detail) + +clients = [] + +if actions != '': + # if specific actions are specified, figure some stuff out now + # e.g., oa1:ra1:ca1,oa1:ra1:ca1 which is list of 0's actions, then 1's, then... + cactions = actions.split(',') + if numclients != len(cactions): + numclients = len(cactions) + i = 0 + for clist in cactions: + clients.append(client('c%d' % i, i, s, f, [], len(clist), clist, solve, detail)) + i += 1 +else: + # else, make random clients + for i in range(numclients): + clients.append(client('c%d' % i, i, s, f, [readratio, 1.0], numsteps, '', solve, detail)) + +# tell server about these clients +s.setclients(clients) + +# init print out for clients +print '%12s' % 'Server', '%12s' % ' ', +for c in clients: + print '%13s' % c.getname(), '%13s' % ' ', +print '' + +# main loop +# +# over time, pick a random client +# have it do one thing, show what happens +# move on to next and so forth + +s.filestats(True) + +# for use with specific schedule +schedcurr = 0 + +# check for legal schedule (must include all clients) +if schedule != '': + for i in range(len(clients)): + cnt = 0 + for j in range(len(schedule)): + curr = schedule[j] + if int(curr) == i: + cnt += 1 + zassert(cnt != 0, 'main: client %d not in schedule:%s, which would never terminate' % (i, schedule)) + +# RUN the schedule (either random or specified by user) +numrunning = len(clients) +while numrunning > 0: + if schedule == '': + c = pickrand(clients) + else: + idx = int(schedule[schedcurr]) + # print 'SCHEDULE DEBUG:: schedule:', schedule, 'schedcurr', schedcurr, 'index', idx + c = clients[idx] + schedcurr += 1 + if schedcurr == len(schedule): + schedcurr = 0 + rc = c.step(clients.index(c)) + if rc == 0: + numrunning -= 1 + + +s.filestats(solve) + +if printstats: + s.stats() + for c in clients: + c.stats() + diff --git a/related_info/ostep/ostep15-disk/disk-precise.py b/related_info/ostep/ostep15-disk/disk-precise.py new file mode 100755 index 0000000..3531506 --- /dev/null +++ b/related_info/ostep/ostep15-disk/disk-precise.py @@ -0,0 +1,719 @@ +#! /usr/bin/env python + +from Tkinter import * +from types import * +import math, random, time, sys, os +from optparse import OptionParser +from decimal import * + +MAXTRACKS = 1000 + +# states that a request/disk go through +STATE_NULL = 0 +STATE_SEEK = 1 +STATE_ROTATE = 2 +STATE_XFER = 3 +STATE_DONE = 4 + +# +# TODO +# XXX transfer time +# XXX satf +# XXX skew +# XXX scheduling window +# XXX sstf +# XXX specify requests vs. random requests in range +# XXX add new requests as old ones complete (starvation) +# XXX run in non-graphical mode +# XXX better graphical display (show key, long lists of requests, more timings on screen) +# XXX be able to do "pure" sequential +# XXX add more blocks around outer tracks (zoning) +# XXX simple flag to make scheduling window a fairness window (-F) +# new algs to scan and c-scan the disk? +# + +class Disk: + def __init__(self, addr, addrDesc, lateAddr, lateAddrDesc, + policy, seekSpeed, rotateSpeed, skew, window, compute, + graphics, zoning): + self.addr = addr + self.addrDesc = addrDesc + self.lateAddr = lateAddr + self.lateAddrDesc = lateAddrDesc + self.policy = policy + self.seekSpeed = Decimal(seekSpeed) + self.rotateSpeed = Decimal(rotateSpeed) + self.skew = skew + self.window = window + self.compute = compute + self.graphics = graphics + self.zoning = zoning + + # figure out zones first, to figure out the max possible request + self.InitBlockLayout() + + # figure out requests + random.seed(options.seed) + self.requests = self.MakeRequests(self.addr, self.addrDesc) + self.lateRequests = self.MakeRequests(self.lateAddr, self.lateAddrDesc) + + # graphical startup + self.width = 500 + if self.graphics: + self.root = Tk() + tmpLen = len(self.requests) + if len(self.lateRequests) > 0: + tmpLen += len(self.lateRequests) + self.canvas = Canvas(self.root, width=410, height=460 + ((tmpLen / 20) * 20)) + self.canvas.pack() + + # fairness stuff + if self.policy == 'BSATF' and self.window != -1: + self.fairWindow = self.window + else: + self.fairWindow = -1 + + print 'REQUESTS', self.requests + print '' + + # for late requests + self.lateCount = 0 + if len(self.lateRequests) > 0: + print 'LATE REQUESTS', self.lateRequests + print '' + + if self.compute == False: + print '' + print 'For the requests above, compute the seek, rotate, and transfer times.' + print 'Use -c or the graphical mode (-G) to see the answers.' + print '' + + # BINDINGS + if self.graphics: + self.root.bind('s', self.Start) + self.root.bind('p', self.Pause) + self.root.bind('q', self.Exit) + + # TRACK INFO + self.tracks = {} + self.trackWidth = 40 + self.tracks[0] = 140 + self.tracks[1] = self.tracks[0] - self.trackWidth + self.tracks[2] = self.tracks[1] - self.trackWidth + + if (self.seekSpeed > 1 and self.trackWidth % self.seekSpeed != 0): + print 'Seek speed (%d) must divide evenly into track width (%d)' % (self.seekSpeed, self.trackWidth) + sys.exit(1) + if self.seekSpeed < 1: + x = (self.trackWidth / self.seekSpeed) + y = int(float(self.trackWidth) / float(self.seekSpeed)) + if float(x) != float(y): + print 'Seek speed (%d) must divide evenly into track width (%d)' % (self.seekSpeed, self.trackWidth) + sys.exit(1) + + # DISK SURFACE + self.cx = self.width/2.0 + self.cy = self.width/2.0 + if self.graphics: + self.canvas.create_rectangle(self.cx-175, 30, self.cx - 20, 80, fill='gray', outline='black') + self.platterSize = 320 + ps2 = self.platterSize / 2.0 + if self.graphics: + self.canvas.create_oval(self.cx-ps2, self.cy-ps2, self.cx+ps2, self.cy + ps2, fill='darkgray', outline='black') + for i in range(len(self.tracks)): + t = self.tracks[i] - (self.trackWidth / 2.0) + if self.graphics: + self.canvas.create_oval(self.cx - t, self.cy - t, self.cx + t, self.cy + t, fill='', outline='black', width=1.0) + + # SPINDLE + self.spindleX = self.cx + self.spindleY = self.cy + if self.graphics: + self.spindleID = self.canvas.create_oval(self.spindleX-3, self.spindleY-3, self.spindleX+3, self.spindleY+3, fill='orange', outline='black') + + # DISK ARM + self.armTrack = 0 + self.armSpeedBase = float(seekSpeed) + self.armSpeed = float(seekSpeed) + + distFromSpindle = self.tracks[self.armTrack] + self.armWidth = 20 + self.headWidth = 10 + + self.armX = self.spindleX - (distFromSpindle * math.cos(math.radians(0))) + self.armX1 = self.armX - self.armWidth + self.armX2 = self.armX + self.armWidth + self.armY1 = 50.0 + self.armY2 = self.width / 2.0 + + self.headX1 = self.armX - self.headWidth + self.headX2 = self.armX + self.headWidth + self.headY1 = (self.width/2.0) - self.headWidth + self.headY2 = (self.width/2.0) + self.headWidth + + if self.graphics: + self.armID = self.canvas.create_rectangle(self.armX1, self.armY1, self.armX2, self.armY2, fill='gray', outline='black') + self.headID = self.canvas.create_rectangle(self.headX1, self.headY1, self.headX2, self.headY2, fill='gray', outline='black') + + self.targetSize = 10.0 + if self.graphics: + sz = self.targetSize + self.targetID = self.canvas.create_oval(self.armX1-sz, self.armY1-sz, self.armX1+sz, self.armY1+sz, fill='orange', outline='') + + # IO QUEUE + self.queueX = 20 + self.queueY = 450 + + self.requestCount = 0 + self.requestQueue = [] + self.requestState = [] + self.queueBoxSize = 20 + self.queueBoxID = {} + self.queueTxtID = {} + + # draw each box + for index in range(len(self.requests)): + self.AddQueueEntry(int(self.requests[index]), index) + if self.graphics: + self.canvas.create_text(self.queueX - 5, self.queueY - 20, anchor='w', text='Queue:') + + # scheduling window + self.currWindow = self.window + + # draw current limits of queue + if self.graphics: + self.windowID = -1 + self.DrawWindow() + + # initial scheduling info + self.currentIndex = -1 + self.currentBlock = -1 + + # initial state of disk (vs seeking, rotating, transferring) + self.state = STATE_NULL + + # DRAW BLOCKS on the TRACKS + for bid in range(len(self.blockInfoList)): + (track, angle, name) = self.blockInfoList[bid] + if self.graphics: + distFromSpindle = self.tracks[track] + xc = self.spindleX + (distFromSpindle * math.cos(math.radians(angle))) + yc = self.spindleY + (distFromSpindle * math.sin(math.radians(angle))) + cid = self.canvas.create_text(xc, yc, text=name, anchor='center') + else: + cid = -1 + self.blockInfoList[bid] = (track, angle, name, cid) + + # angle of rotation + self.angle = Decimal(0.0) + + # TIME INFO + if self.graphics: + self.timeID = self.canvas.create_text(10, 10, text='Time: 0.00', anchor='w') + self.canvas.create_rectangle(95,0,200,18, fill='orange', outline='orange') + self.seekID = self.canvas.create_text(100, 10, text='Seek: 0.00', anchor='w') + self.canvas.create_rectangle(195,0,300,18, fill='lightblue', outline='lightblue') + self.rotID = self.canvas.create_text(200, 10, text='Rotate: 0.00', anchor='w') + self.canvas.create_rectangle(295,0,400,18, fill='green', outline='green') + self.xferID = self.canvas.create_text(300, 10, text='Transfer: 0.00', anchor='w') + self.canvas.create_text(320, 40, text='"s" to start', anchor='w') + self.canvas.create_text(320, 60, text='"p" to pause', anchor='w') + self.canvas.create_text(320, 80, text='"q" to quit', anchor='w') + self.timer = 0 + + # STATS + self.seekTotal = 0.0 + self.rotTotal = 0.0 + self.xferTotal = 0.0 + + # set up animation loop + if self.graphics: + self.doAnimate = True + else: + self.doAnimate = False + self.isDone = False + + # call this to start simulation + def Go(self): + if options.graphics: + self.root.mainloop() + else: + self.GetNextIO() + while self.isDone == False: + self.Animate() + + # crappy error message + def PrintAddrDescMessage(self, value): + print 'Bad address description (%s)' % value + print 'The address description must be a comma-separated list of length three, without spaces.' + print 'For example, "10,100,0" would indicate that 10 addresses should be generated, with' + print '100 as the maximum value, and 0 as the minumum. A max of -1 means just use the highest' + print 'possible value as the max address to generate.' + sys.exit(1) + + # + # ZONES AND BLOCK LAYOUT + # + def InitBlockLayout(self): + self.blockInfoList = [] + self.blockToTrackMap = {} + self.blockToAngleMap = {} + self.tracksBeginEnd = {} + self.blockAngleOffset = [] + + zones = self.zoning.split(',') + assert(len(zones) == 3) + for i in range(len(zones)): + self.blockAngleOffset.append(int(zones[i]) / 2) + + track = 0 # outer track + angleOffset = 2 * self.blockAngleOffset[track] + for angle in range(0, 360, angleOffset): + block = angle / angleOffset + self.blockToTrackMap[block] = track + self.blockToAngleMap[block] = angle + self.blockInfoList.append((track, angle, block)) + self.tracksBeginEnd[track] = (0, block) + pblock = block + 1 + + track = 1 # middle track + skew = self.skew + angleOffset = 2 * self.blockAngleOffset[track] + for angle in range(0, 360, angleOffset): + block = (angle / angleOffset) + pblock + self.blockToTrackMap[block] = track + self.blockToAngleMap[block] = angle + (angleOffset * skew) + self.blockInfoList.append((track, angle + (angleOffset * skew), block)) + self.tracksBeginEnd[track] = (pblock, block) + pblock = block + 1 + + track = 2 # inner track + skew = 2 * self.skew + angleOffset = 2 * self.blockAngleOffset[track] + for angle in range(0, 360, angleOffset): + block = (angle / angleOffset) + pblock + self.blockToTrackMap[block] = track + self.blockToAngleMap[block] = angle + (angleOffset * skew) + self.blockInfoList.append((track, angle + (angleOffset * skew), block)) + self.tracksBeginEnd[track] = (pblock, block) + self.maxBlock = pblock + # print 'MAX BLOCK:', self.maxBlock + + # adjust angle to starting position relative + for i in self.blockToAngleMap: + self.blockToAngleMap[i] = (self.blockToAngleMap[i] + 180) % 360 + + # print 'btoa map', self.blockToAngleMap + # print 'btot map', self.blockToTrackMap + # print 'bao', self.blockAngleOffset + + def MakeRequests(self, addr, addrDesc): + (numRequests, maxRequest, minRequest) = (0, 0, 0) + if addr == '-1': + # first extract values from descriptor + desc = addrDesc.split(',') + if len(desc) != 3: + self.PrintAddrDescMessage(addrDesc) + (numRequests, maxRequest, minRequest) = (int(desc[0]), int(desc[1]), int(desc[2])) + if maxRequest == -1: + maxRequest = self.maxBlock + # now make list + tmpList = [] + for i in range(numRequests): + tmpList.append(int(random.random() * maxRequest) + minRequest) + return tmpList + else: + return addr.split(',') + + # + # BUTTONS + # + def Start(self, event): + self.GetNextIO() + self.doAnimate = True + self.Animate() + + def Pause(self, event): + if self.doAnimate == False: + self.doAnimate = True + else: + self.doAnimate = False + + def Exit(self, event): + sys.exit(0) + + # + # CORE SIMULATION and ANIMATION + # + def UpdateTime(self): + if self.graphics: + self.canvas.itemconfig(self.timeID, text='Time: ' + str(self.timer)) + self.canvas.itemconfig(self.seekID, text='Seek: ' + str(self.seekTotal)) + self.canvas.itemconfig(self.rotID, text='Rotate: ' + str(self.rotTotal)) + self.canvas.itemconfig(self.xferID, text='Transfer: ' + str(self.xferTotal)) + + def AddRequest(self, block): + self.AddQueueEntry(block, len(self.requestQueue)) + + def QueueMap(self, index): + numPerRow = 400 / self.queueBoxSize + return (index % numPerRow, index / numPerRow) + + def DrawWindow(self): + if self.window == -1: + return + (col, row) = self.QueueMap(self.currWindow) + if col == 0: + (col, row) = (20, row - 1) + if self.windowID != -1: + self.canvas.delete(self.windowID) + self.windowID = self.canvas.create_line(self.queueX + (col * 20) - 10, self.queueY - 13 + (row * 20), + self.queueX + (col * 20) - 10, self.queueY + 13 + (row * 20), width=2) + + def AddQueueEntry(self, block, index): + self.requestQueue.append((block, index)) + self.requestState.append(STATE_NULL) + if self.graphics: + (col, row) = self.QueueMap(index) + sizeHalf = self.queueBoxSize / 2.0 + (cx, cy) = (self.queueX + (col * self.queueBoxSize), self.queueY + (row * self.queueBoxSize)) + self.queueBoxID[index] = self.canvas.create_rectangle(cx - sizeHalf, cy - sizeHalf, cx + sizeHalf, cy + sizeHalf, fill='white') + self.queueTxtID[index] = self.canvas.create_text(cx, cy, anchor='center', text=str(block)) + + def SwitchColors(self, c): + if self.graphics: + self.canvas.itemconfig(self.queueBoxID[self.currentIndex], fill=c) + self.canvas.itemconfig(self.targetID, fill=c) + + def SwitchState(self, newState): + self.state = newState + self.requestState[self.currentIndex] = newState + + def RadiallyCloseTo(self, a1, a2): + if a1 > a2: + v = a1 - a2 + else: + v = a2 - a1 + if v < self.rotateSpeed: + return True + return False + + def DoneWithTransfer(self): + angleOffset = self.blockAngleOffset[self.armTrack] + # if int(self.angle) == (self.blockToAngleMap[self.currentBlock] + angleOffset) % 360: + if self.RadiallyCloseTo(self.angle, Decimal((self.blockToAngleMap[self.currentBlock] + angleOffset) % 360)): + # print 'END TRANSFER', self.angle, self.timer + self.SwitchState(STATE_DONE) + self.requestCount += 1 + return True + return False + + def DoneWithRotation(self): + angleOffset = self.blockAngleOffset[self.armTrack] + # XXX there is a weird bug in here + # print self.timer, 'ROTATE:: ', self.currentBlock, 'currangle: ', self.angle, ' - mapangle: ', self.blockToAngleMap[self.currentBlock] + # print ' angleOffset ', angleOffset + # print ' blockMap ', (self.blockToAngleMap[self.currentBlock] - angleOffset) % 360 + # print ' self.angle ', self.angle, int(self.angle) + # if int(self.angle) == (self.blockToAngleMap[self.currentBlock] - angleOffset) % 360: + if self.RadiallyCloseTo(self.angle, Decimal((self.blockToAngleMap[self.currentBlock] - angleOffset) % 360)): + self.SwitchState(STATE_XFER) + # print ' --> DONE WITH ROTATION!', self.timer + return True + return False + + def PlanSeek(self, track): + self.seekBegin = self.timer + self.SwitchColors('orange') + self.SwitchState(STATE_SEEK) + if track == self.armTrack: + self.rotBegin = self.timer + self.SwitchColors('lightblue') + self.SwitchState(STATE_ROTATE) + return + self.armTarget = track + self.armTargetX1 = self.spindleX - self.tracks[track] - (self.trackWidth / 2.0) + if track >= self.armTrack: + self.armSpeed = self.armSpeedBase + else: + self.armSpeed = - self.armSpeedBase + + def DoneWithSeek(self): + # move the disk arm + self.armX1 += self.armSpeed + self.armX2 += self.armSpeed + self.headX1 += self.armSpeed + self.headX2 += self.armSpeed + # update it on screen + if self.graphics: + self.canvas.coords(self.armID, self.armX1, self.armY1, self.armX2, self.armY2) + self.canvas.coords(self.headID, self.headX1, self.headY1, self.headX2, self.headY2) + # check if done + if (self.armSpeed > 0.0 and self.armX1 >= self.armTargetX1) or (self.armSpeed < 0.0 and self.armX1 <= self.armTargetX1): + self.armTrack = self.armTarget + return True + return False + + def DoSATF(self, rList): + minBlock = -1 + minIndex = -1 + minEst = -1 + + # print '**** DoSATF ****', rList + for (block, index) in rList: + if self.requestState[index] == STATE_DONE: + continue + track = self.blockToTrackMap[block] + angle = self.blockToAngleMap[block] + # print 'track', track, 'angle', angle + + # estimate seek time + dist = int(math.fabs(self.armTrack - track)) + seekEst = Decimal(self.trackWidth / self.armSpeedBase) * dist + + # estimate rotate time + angleOffset = self.blockAngleOffset[track] + angleAtArrival = (Decimal(self.angle) + (seekEst * self.rotateSpeed)) + while angleAtArrival > 360.0: + angleAtArrival -= 360.0 + rotDist = Decimal((angle - angleOffset) - angleAtArrival) + while rotDist > 360.0: + rotDist -= Decimal(360.0) + while rotDist < 0.0: + rotDist += Decimal(360.0) + rotEst = rotDist / self.rotateSpeed + + # finally, transfer + xferEst = (Decimal(angleOffset) * Decimal(2.0)) / self.rotateSpeed + + totalEst = seekEst + rotEst + xferEst + + # should probably pick one on same track in case of a TIE + if minEst == -1 or totalEst < minEst: + minEst = totalEst + minBlock = block + minIndex = index + # END loop + + # when done + self.totalEst = minEst + assert(minBlock != -1) + assert(minIndex != -1) + return (minBlock, minIndex) + + # + # actually doesn't quite do SSTF + # just finds all the blocks on the nearest track + # (whatever that may be) and returns it as a list + # + def DoSSTF(self, rList): + minDist = MAXTRACKS + minBlock = -1 + trackList = [] # all the blocks on a track + + for (block, index) in rList: + if self.requestState[index] == STATE_DONE: + continue + track = self.blockToTrackMap[block] + dist = int(math.fabs(self.armTrack - track)) + if dist < minDist: + trackList = [] + trackList.append((block, index)) + minDist = dist + elif dist == minDist: + trackList.append((block, index)) + assert(trackList != []) + return trackList + + def UpdateWindow(self): + if self.fairWindow == -1 and self.currWindow > 0 and self.currWindow < len(self.requestQueue): + self.currWindow += 1 + if self.graphics: + self.DrawWindow() + + def GetWindow(self): + if self.currWindow <= -1: + return len(self.requestQueue) + else: + if self.fairWindow != -1: + if self.requestCount > 0 and (self.requestCount % self.fairWindow == 0): + self.currWindow = self.currWindow + self.fairWindow + if self.currWindow > len(self.requestQueue): + self.currWindow = len(self.requestQueue) + if self.graphics: + self.DrawWindow() + return self.currWindow + else: + return self.currWindow + + def GetNextIO(self): + # check if done: if so, print stats and end animation + if self.requestCount == len(self.requestQueue): + self.UpdateTime() + self.PrintStats() + self.doAnimate = False + self.isDone = True + return + + # do policy: should set currentBlock, + if self.policy == 'FIFO': + (self.currentBlock, self.currentIndex) = self.requestQueue[self.requestCount] + self.DoSATF(self.requestQueue[self.requestCount:self.requestCount+1]) + elif self.policy == 'SATF' or self.policy == 'BSATF': + (self.currentBlock, self.currentIndex) = self.DoSATF(self.requestQueue[0:self.GetWindow()]) + elif self.policy == 'SSTF': + # first, find all the blocks on a given track (given window constraints) + trackList = self.DoSSTF(self.requestQueue[0:self.GetWindow()]) + # then, do SATF on those blocks (otherwise, will not do them in obvious order) + (self.currentBlock, self.currentIndex) = self.DoSATF(trackList) + else: + print 'policy (%s) not implemented' % self.policy + sys.exit(1) + + # once best block is decided, go ahead and do the seek + self.PlanSeek(self.blockToTrackMap[self.currentBlock]) + + # add another block? + if len(self.lateRequests) > 0 and self.lateCount < len(self.lateRequests): + self.AddRequest(self.lateRequests[self.lateCount]) + self.lateCount += 1 + + def Animate(self): + if self.graphics == True and self.doAnimate == False: + self.root.after(20, self.Animate) + return + + # timer + self.timer += 1 + self.UpdateTime() + + # see which blocks are rotating on the disk + # print 'SELF ANGLE', self.angle + self.angle = Decimal(self.angle + self.rotateSpeed) + if self.angle >= 360.0: + self.angle = Decimal(0.0) + + # move the blocks + if self.graphics: + for (track, angle, name, cid) in self.blockInfoList: + distFromSpindle = self.tracks[track] + na = angle - self.angle + xc = self.spindleX + (distFromSpindle * math.cos(math.radians(na))) + yc = self.spindleY + (distFromSpindle * math.sin(math.radians(na))) + if self.graphics: + self.canvas.coords(cid, xc, yc) + if self.currentBlock == name: + sz = self.targetSize + self.canvas.coords(self.targetID, xc-sz, yc-sz, xc+sz, yc+sz) + + # move the arm OR wait for a rotational delay + if self.state == STATE_SEEK: + if self.DoneWithSeek(): + self.rotBegin = self.timer + self.SwitchState(STATE_ROTATE) + self.SwitchColors('lightblue') + if self.state == STATE_ROTATE: + # check for read (disk arm must be settled) + if self.DoneWithRotation(): + self.xferBegin = self.timer + self.SwitchState(STATE_XFER) + self.SwitchColors('green') + if self.state == STATE_XFER: + if self.DoneWithTransfer(): + self.DoRequestStats() + self.SwitchState(STATE_DONE) + self.SwitchColors('red') + self.UpdateWindow() + currentBlock = self.currentBlock + self.GetNextIO() + nextBlock = self.currentBlock + if self.blockToTrackMap[currentBlock] == self.blockToTrackMap[nextBlock]: + if (currentBlock == self.tracksBeginEnd[self.armTrack][1] and nextBlock == self.tracksBeginEnd[self.armTrack][0]) or (currentBlock + 1 == nextBlock): + # need a special case here: to handle when we stay in transfer mode + (self.rotBegin, self.seekBegin, self.xferBegin) = (self.timer, self.timer, self.timer) + self.SwitchState(STATE_XFER) + self.SwitchColors('green') + + + + # make sure to keep the animation going! + if self.graphics: + self.root.after(20, self.Animate) + + def DoRequestStats(self): + seekTime = self.rotBegin - self.seekBegin + rotTime = self.xferBegin - self.rotBegin + xferTime = self.timer - self.xferBegin + totalTime = self.timer - self.seekBegin + + if self.compute == True: + print 'Block: %3d Seek:%3d Rotate:%3d Transfer:%3d Total:%4d' % (self.currentBlock, seekTime, rotTime, xferTime, totalTime) + + # if int(totalTime) != int(self.totalEst): + # print 'INTERNAL ERROR: estimate was', self.totalEst, 'whereas actual time to access block was', totalTime + # print 'Please report this bug and as much information as possible so as to make it easy to recreate. Thanks!' + + # update stats + self.seekTotal += seekTime + self.rotTotal += rotTime + self.xferTotal += xferTime + + + + def PrintStats(self): + if self.compute == True: + print '\nTOTALS Seek:%3d Rotate:%3d Transfer:%3d Total:%4d\n' % (self.seekTotal, self.rotTotal, self.xferTotal, self.timer) + +# END: class Disk + + + +# +# MAIN SIMULATOR +# +parser = OptionParser() +parser.add_option('-s', '--seed', default='0', help='Random seed', action='store', type='int', dest='seed') +parser.add_option('-a', '--addr', default='-1', help='Request list (comma-separated) [-1 -> use addrDesc]', action='store', type='string', dest='addr') +parser.add_option('-A', '--addrDesc', default='5,-1,0', help='Num requests, max request (-1->all), min request', action='store', type='string', dest='addrDesc') +parser.add_option('-S', '--seekSpeed', default='1', help='Speed of seek', action='store', type='string', dest='seekSpeed') +parser.add_option('-R', '--rotSpeed', default='1', help='Speed of rotation', action='store', type='string', dest='rotateSpeed') +parser.add_option('-p', '--policy', default='FIFO', help='Scheduling policy (FIFO, SSTF, SATF, BSATF)', action='store', type='string', dest='policy') +parser.add_option('-w', '--schedWindow', default=-1, help='Size of scheduling window (-1 -> all)', action='store', type='int', dest='window') +parser.add_option('-o', '--skewOffset', default=0, help='Amount of skew (in blocks)', action='store', type='int', dest='skew') +parser.add_option('-z', '--zoning', default='30,30,30', help='Angles between blocks on outer,middle,inner tracks', action='store', type='string', dest='zoning') +parser.add_option('-G', '--graphics', default=False, help='Turn on graphics', action='store_true', dest='graphics') +parser.add_option('-l', '--lateAddr', default='-1', help='Late: request list (comma-separated) [-1 -> random]', action='store', type='string', dest='lateAddr') +parser.add_option('-L', '--lateAddrDesc', default='0,-1,0', help='Num requests, max request (-1->all), min request', action='store', type='string', dest='lateAddrDesc') +parser.add_option('-c', '--compute', default=False, help='Compute the answers', action='store_true', dest='compute') +(options, args) = parser.parse_args() + +print 'OPTIONS seed', options.seed +print 'OPTIONS addr', options.addr +print 'OPTIONS addrDesc', options.addrDesc +print 'OPTIONS seekSpeed', options.seekSpeed +print 'OPTIONS rotateSpeed', options.rotateSpeed +print 'OPTIONS skew', options.skew +print 'OPTIONS window', options.window +print 'OPTIONS policy', options.policy +print 'OPTIONS compute', options.compute +print 'OPTIONS graphics', options.graphics +print 'OPTIONS zoning', options.zoning +print 'OPTIONS lateAddr', options.lateAddr +print 'OPTIONS lateAddrDesc', options.lateAddrDesc +print '' + +if options.window == 0: + print 'Scheduling window (%d) must be positive or -1 (which means a full window)' % options.window + sys.exit(1) + +if options.graphics and options.compute == False: + print '\nWARNING: Setting compute flag to True, as graphics are on\n' + options.compute = True + +# set up simulator info +d = Disk(addr=options.addr, addrDesc=options.addrDesc, lateAddr=options.lateAddr, lateAddrDesc=options.lateAddrDesc, + policy=options.policy, seekSpeed=Decimal(options.seekSpeed), rotateSpeed=Decimal(options.rotateSpeed), + skew=options.skew, window=options.window, compute=options.compute, graphics=options.graphics, zoning=options.zoning) + +# run simulation +d.Go() diff --git a/related_info/ostep/ostep15-disk/disk.md b/related_info/ostep/ostep15-disk/disk.md new file mode 100644 index 0000000..d6f8fb1 --- /dev/null +++ b/related_info/ostep/ostep15-disk/disk.md @@ -0,0 +1,133 @@ + +This homework uses disk.py to familiarize you with how a modern hard +drive works. It has a lot of different options, and unlike most of the other +simulations, has a graphical animator to show you exactly what happens when +the disk is in action. + +[Note: there is also an experimental program, 'disk-precise.py', included +in the download. This version of the simulator uses the python Decimal +package for precise floating point computation, thus giving slightly +better answers in some corner cases than 'disk.py'. However, it has +not been very carefully tested, so use at your own caution.] + +Let's do a simple example first. To run the simulator and compute some basic +seek, rotation, and transfer times, you first have to give a list of requests +to the simulator. This can either be done by specifying the exact requests, or +by having the simulator generate some randomly. + +We'll start by specifying a list of requests ourselves. Let's do a single +request first: + +prompt> disk.py -a 10 + +At this point you'll see: + +... +REQUESTS [br '10'] + +For the requests above, compute the seek, rotate, and transfer times. +Use -c or the graphical mode (-G) to see the answers. + + +To be able to compute the seek, rotation, and transfer times for this request, +you'll have to know a little more information about the layout of sectors, the +starting position of the disk head, and so forth. To see much of this +information, run the simulator in graphical mode (-G): + +prompt> disk.py -a 10 -G + + +At this point, a window should appear with our simple disk on it. +The disk head is positioned on the outside track, halfway through sector 6. +As you can see, sector 10 (our example sector) is on the same track, about a +third of the way around. The direction of rotation is counter-clockwise. +To run the simulation, press the "s" key while the simulator window is +highlighted. + +When the simulation completes, you should be able to see that the disk spent +105 time units in rotation and 30 in transfer in order to access sector 10, +with no seek time. Press "q" to close the simulator window. + +To calculate this (instead of just running the simulation), you would need to +know a few details about the disk. First, the rotational speed is by default +set to 1 degree per time unit. Thus, to make a complete revolution, it takes +360 time units. Second, transfer begins and ends at the halfway point between +sectors. Thus, to read sector 10, the transfer begins halfway between 9 and 10, +and ends halfway between 10 and 11. Finally, in the default disk, there are +12 sectors per track, meaning that each sector takes up 30 degrees of the +rotational space. Thus, to read a sector, it takes 30 time units (given our +default speed of rotation). + +With this information in hand, you now should be able to compute the seek, +rotation, and transfer times for accessing sector 10. Because the head starts +on the same track as 10, there is no seek time. Because the disk rotates at +1 degree / time unit, it takes 105 time units to get to the beginning of sector +10, halfway between 9 and 10 (note that it is exactly 90 degrees to the middle +of sector 9, and another 15 to the halfway point). Finally, to transfer the +sector takes 30 time units. + +Now let's do a slightly more complex example: + +prompt> disk.py -a 10,11 -G + + +In this case, we're transferring two sectors, 10 and 11. How long will it take? +Try guessing before running the simulation! + +As you probably guessed, this simulation takes just 30 time units longer, to +transfer the next sector 11. Thus, the seek and rotate times remain the same, +but the transfer time for the requests is doubled. You can in fact see these +sums across the top of the simulator window; they also get printed out to the +console as follows: + +... +Sector: 10 Seek: 0 Rotate:105 Transfer: 30 Total: 135 +Sector: 11 Seek: 0 Rotate: 0 Transfer: 30 Total: 30 +TOTALS Seek: 0 Rotate:105 Transfer: 60 Total: 165 + + +Now let's do an example with a seek. Try the following set of requests: + +prompt> disk.py -a 10,18 -G + + +To compute how long this will take, you need to know how long a seek will +take. The distance between each track is by default 40 distance units, and the +default rate of seeking is 1 distance unit per unit time. Thus, a seek from +the outer track to the middle track takes 40 time units. + +You'd also have to know the scheduling policy. The default is FIFO, though, so +for now you can just compute the request times assuming the processing order +matches the list specified via the "-a" flag. + +To compute how long it will take the disk to service these requests, we first +compute how long it takes to access sector 10, which we know from above to be +135 time units (105 rotating, 30 transferring). Once this request is complete, +the disk begins to seek to the middle track where sector 18 lies, taking 40 +time units. Then the disk rotates to sector 18, and transfers it for 30 time +units, thus completing the simulation. But how long does this final rotation +take? + +To compute the rotational delay for 18, first figure out how long the disk +would take to rotate from the end of the access to sector 10 to the beginning +of the access to sector 18, assuming a zero-cost seek. As you can see from the +simulator, sector 10 on the outer track is lined up with sector 22 on the middle +track, and there are 7 sectors separating 22 from 18 (23, 12, 13, 14, 15, 16, +and 17, as the disk spins counter-clockwise). Rotating through 7 sectors takes +210 time units (30 per sector). However, the first part of this rotation is +actually spent seeking to the middle track, for 40 time units. Thus, the +actual rotational delay for accessing sector 18 is 210 minus 40, or 170 time +units. Run the simulator to see this for yourself; note that you can run +without graphics and with the "-c" flag to just see the results without +seeing the graphics. + +prompt> ./disk.py -a 10,18 -c +... +Sector: 10 Seek: 0 Rotate:105 Transfer: 30 Total: 135 +Sector: 18 Seek: 40 Rotate:170 Transfer: 30 Total: 240 +TOTALS Seek: 40 Rotate:275 Transfer: 60 Total: 375 + +You should now have a basic idea of how the simulator works. The questions +below will explore some of the different options, to better help you build a +model of how a disk really works. + diff --git a/related_info/ostep/ostep15-disk/disk.py b/related_info/ostep/ostep15-disk/disk.py new file mode 100755 index 0000000..97754b4 --- /dev/null +++ b/related_info/ostep/ostep15-disk/disk.py @@ -0,0 +1,730 @@ +#! /usr/bin/env python + +from Tkinter import * +from types import * +import math, random, time, sys, os +from optparse import OptionParser + +MAXTRACKS = 1000 + +# states that a request/disk go through +STATE_NULL = 0 +STATE_SEEK = 1 +STATE_ROTATE = 2 +STATE_XFER = 3 +STATE_DONE = 4 + +# +# TODO +# XXX transfer time +# XXX satf +# XXX skew +# XXX scheduling window +# XXX sstf +# XXX specify requests vs. random requests in range +# XXX add new requests as old ones complete (starvation) +# XXX run in non-graphical mode +# XXX better graphical display (show key, long lists of requests, more timings on screen) +# XXX be able to do "pure" sequential +# XXX add more blocks around outer tracks (zoning) +# XXX simple flag to make scheduling window a fairness window (-F) +# new algs to scan and c-scan the disk? +# + +class Disk: + def __init__(self, addr, addrDesc, lateAddr, lateAddrDesc, + policy, seekSpeed, rotateSpeed, skew, window, compute, + graphics, zoning): + self.addr = addr + self.addrDesc = addrDesc + self.lateAddr = lateAddr + self.lateAddrDesc = lateAddrDesc + self.policy = policy + self.seekSpeed = seekSpeed + self.rotateSpeed = rotateSpeed + self.skew = skew + self.window = window + self.compute = compute + self.graphics = graphics + self.zoning = zoning + + # figure out zones first, to figure out the max possible request + self.InitBlockLayout() + + # figure out requests + random.seed(options.seed) + self.requests = self.MakeRequests(self.addr, self.addrDesc) + self.lateRequests = self.MakeRequests(self.lateAddr, self.lateAddrDesc) + + # graphical startup + self.width = 500 + if self.graphics: + self.root = Tk() + tmpLen = len(self.requests) + if len(self.lateRequests) > 0: + tmpLen += len(self.lateRequests) + self.canvas = Canvas(self.root, width=410, height=460 + ((tmpLen / 20) * 20)) + self.canvas.pack() + + # fairness stuff + if self.policy == 'BSATF' and self.window != -1: + self.fairWindow = self.window + else: + self.fairWindow = -1 + + print 'REQUESTS', self.requests + print '' + + # for late requests + self.lateCount = 0 + if len(self.lateRequests) > 0: + print 'LATE REQUESTS', self.lateRequests + print '' + + if self.compute == False: + print '' + print 'For the requests above, compute the seek, rotate, and transfer times.' + print 'Use -c or the graphical mode (-G) to see the answers.' + print '' + + # BINDINGS + if self.graphics: + self.root.bind('s', self.Start) + self.root.bind('p', self.Pause) + self.root.bind('q', self.Exit) + + # TRACK INFO + self.tracks = {} + self.trackWidth = 40 + self.tracks[0] = 140 + self.tracks[1] = self.tracks[0] - self.trackWidth + self.tracks[2] = self.tracks[1] - self.trackWidth + + if (self.seekSpeed > 1 and self.trackWidth % self.seekSpeed != 0): + print 'Seek speed (%d) must divide evenly into track width (%d)' % (self.seekSpeed, self.trackWidth) + sys.exit(1) + if self.seekSpeed < 1: + x = (self.trackWidth / self.seekSpeed) + y = int(float(self.trackWidth) / float(self.seekSpeed)) + if float(x) != float(y): + print 'Seek speed (%d) must divide evenly into track width (%d)' % (self.seekSpeed, self.trackWidth) + sys.exit(1) + + # DISK SURFACE + self.cx = self.width/2.0 + self.cy = self.width/2.0 + if self.graphics: + self.canvas.create_rectangle(self.cx-175, 30, self.cx - 20, 80, fill='gray', outline='black') + self.platterSize = 320 + ps2 = self.platterSize / 2.0 + if self.graphics: + self.canvas.create_oval(self.cx-ps2, self.cy-ps2, self.cx+ps2, self.cy + ps2, fill='darkgray', outline='black') + for i in range(len(self.tracks)): + t = self.tracks[i] - (self.trackWidth / 2.0) + if self.graphics: + self.canvas.create_oval(self.cx - t, self.cy - t, self.cx + t, self.cy + t, fill='', outline='black', width=1.0) + + # SPINDLE + self.spindleX = self.cx + self.spindleY = self.cy + if self.graphics: + self.spindleID = self.canvas.create_oval(self.spindleX-3, self.spindleY-3, self.spindleX+3, self.spindleY+3, fill='orange', outline='black') + + # DISK ARM + self.armTrack = 0 + self.armSpeedBase = float(seekSpeed) + self.armSpeed = float(seekSpeed) + + distFromSpindle = self.tracks[self.armTrack] + self.armWidth = 20 + self.headWidth = 10 + + self.armX = self.spindleX - (distFromSpindle * math.cos(math.radians(0))) + self.armX1 = self.armX - self.armWidth + self.armX2 = self.armX + self.armWidth + self.armY1 = 50.0 + self.armY2 = self.width / 2.0 + + self.headX1 = self.armX - self.headWidth + self.headX2 = self.armX + self.headWidth + self.headY1 = (self.width/2.0) - self.headWidth + self.headY2 = (self.width/2.0) + self.headWidth + + if self.graphics: + self.armID = self.canvas.create_rectangle(self.armX1, self.armY1, self.armX2, self.armY2, fill='gray', outline='black') + self.headID = self.canvas.create_rectangle(self.headX1, self.headY1, self.headX2, self.headY2, fill='gray', outline='black') + + self.targetSize = 10.0 + if self.graphics: + sz = self.targetSize + self.targetID = self.canvas.create_oval(self.armX1-sz, self.armY1-sz, self.armX1+sz, self.armY1+sz, fill='orange', outline='') + + # IO QUEUE + self.queueX = 20 + self.queueY = 450 + + self.requestCount = 0 + self.requestQueue = [] + self.requestState = [] + self.queueBoxSize = 20 + self.queueBoxID = {} + self.queueTxtID = {} + + # draw each box + for index in range(len(self.requests)): + self.AddQueueEntry(int(self.requests[index]), index) + if self.graphics: + self.canvas.create_text(self.queueX - 5, self.queueY - 20, anchor='w', text='Queue:') + + # scheduling window + self.currWindow = self.window + + # draw current limits of queue + if self.graphics: + self.windowID = -1 + self.DrawWindow() + + # initial scheduling info + self.currentIndex = -1 + self.currentBlock = -1 + + # initial state of disk (vs seeking, rotating, transferring) + self.state = STATE_NULL + + # DRAW BLOCKS on the TRACKS + for bid in range(len(self.blockInfoList)): + (track, angle, name) = self.blockInfoList[bid] + if self.graphics: + distFromSpindle = self.tracks[track] + xc = self.spindleX + (distFromSpindle * math.cos(math.radians(angle))) + yc = self.spindleY + (distFromSpindle * math.sin(math.radians(angle))) + cid = self.canvas.create_text(xc, yc, text=name, anchor='center') + else: + cid = -1 + self.blockInfoList[bid] = (track, angle, name, cid) + + # angle of rotation + self.angle = 0.0 + + # TIME INFO + if self.graphics: + self.timeID = self.canvas.create_text(10, 10, text='Time: 0.00', anchor='w') + self.canvas.create_rectangle(95,0,200,18, fill='orange', outline='orange') + self.seekID = self.canvas.create_text(100, 10, text='Seek: 0.00', anchor='w') + self.canvas.create_rectangle(195,0,300,18, fill='lightblue', outline='lightblue') + self.rotID = self.canvas.create_text(200, 10, text='Rotate: 0.00', anchor='w') + self.canvas.create_rectangle(295,0,400,18, fill='green', outline='green') + self.xferID = self.canvas.create_text(300, 10, text='Transfer: 0.00', anchor='w') + self.canvas.create_text(320, 40, text='"s" to start', anchor='w') + self.canvas.create_text(320, 60, text='"p" to pause', anchor='w') + self.canvas.create_text(320, 80, text='"q" to quit', anchor='w') + self.timer = 0 + + # STATS + self.seekTotal = 0.0 + self.rotTotal = 0.0 + self.xferTotal = 0.0 + + # set up animation loop + if self.graphics: + self.doAnimate = True + else: + self.doAnimate = False + self.isDone = False + + # call this to start simulation + def Go(self): + if options.graphics: + self.root.mainloop() + else: + self.GetNextIO() + while self.isDone == False: + self.Animate() + + # crappy error message + def PrintAddrDescMessage(self, value): + print 'Bad address description (%s)' % value + print 'The address description must be a comma-separated list of length three, without spaces.' + print 'For example, "10,100,0" would indicate that 10 addresses should be generated, with' + print '100 as the maximum value, and 0 as the minumum. A max of -1 means just use the highest' + print 'possible value as the max address to generate.' + sys.exit(1) + + # + # ZONES AND BLOCK LAYOUT + # + def InitBlockLayout(self): + self.blockInfoList = [] + self.blockToTrackMap = {} + self.blockToAngleMap = {} + self.tracksBeginEnd = {} + self.blockAngleOffset = [] + + zones = self.zoning.split(',') + assert(len(zones) == 3) + for i in range(len(zones)): + self.blockAngleOffset.append(int(zones[i]) / 2) + + track = 0 # outer track + angleOffset = 2 * self.blockAngleOffset[track] + for angle in range(0, 360, angleOffset): + block = angle / angleOffset + self.blockToTrackMap[block] = track + self.blockToAngleMap[block] = angle + self.blockInfoList.append((track, angle, block)) + self.tracksBeginEnd[track] = (0, block) + pblock = block + 1 + + track = 1 # middle track + skew = self.skew + angleOffset = 2 * self.blockAngleOffset[track] + for angle in range(0, 360, angleOffset): + block = (angle / angleOffset) + pblock + self.blockToTrackMap[block] = track + self.blockToAngleMap[block] = angle + (angleOffset * skew) + self.blockInfoList.append((track, angle + (angleOffset * skew), block)) + self.tracksBeginEnd[track] = (pblock, block) + pblock = block + 1 + + track = 2 # inner track + skew = 2 * self.skew + angleOffset = 2 * self.blockAngleOffset[track] + for angle in range(0, 360, angleOffset): + block = (angle / angleOffset) + pblock + self.blockToTrackMap[block] = track + self.blockToAngleMap[block] = angle + (angleOffset * skew) + self.blockInfoList.append((track, angle + (angleOffset * skew), block)) + self.tracksBeginEnd[track] = (pblock, block) + self.maxBlock = pblock + # print 'MAX BLOCK:', self.maxBlock + + # adjust angle to starting position relative + for i in self.blockToAngleMap: + self.blockToAngleMap[i] = (self.blockToAngleMap[i] + 180) % 360 + + # print 'btoa map', self.blockToAngleMap + # print 'btot map', self.blockToTrackMap + # print 'bao', self.blockAngleOffset + + def MakeRequests(self, addr, addrDesc): + (numRequests, maxRequest, minRequest) = (0, 0, 0) + if addr == '-1': + # first extract values from descriptor + desc = addrDesc.split(',') + if len(desc) != 3: + self.PrintAddrDescMessage(addrDesc) + (numRequests, maxRequest, minRequest) = (int(desc[0]), int(desc[1]), int(desc[2])) + if maxRequest == -1: + maxRequest = self.maxBlock + # now make list + tmpList = [] + for i in range(numRequests): + tmpList.append(int(random.random() * maxRequest) + minRequest) + return tmpList + else: + return addr.split(',') + + # + # BUTTONS + # + def Start(self, event): + self.GetNextIO() + self.doAnimate = True + self.Animate() + + def Pause(self, event): + if self.doAnimate == False: + self.doAnimate = True + else: + self.doAnimate = False + + def Exit(self, event): + sys.exit(0) + + # + # CORE SIMULATION and ANIMATION + # + def UpdateTime(self): + if self.graphics: + self.canvas.itemconfig(self.timeID, text='Time: ' + str(self.timer)) + self.canvas.itemconfig(self.seekID, text='Seek: ' + str(self.seekTotal)) + self.canvas.itemconfig(self.rotID, text='Rotate: ' + str(self.rotTotal)) + self.canvas.itemconfig(self.xferID, text='Transfer: ' + str(self.xferTotal)) + + def AddRequest(self, block): + self.AddQueueEntry(block, len(self.requestQueue)) + + def QueueMap(self, index): + numPerRow = 400 / self.queueBoxSize + return (index % numPerRow, index / numPerRow) + + def DrawWindow(self): + if self.window == -1: + return + (col, row) = self.QueueMap(self.currWindow) + if col == 0: + (col, row) = (20, row - 1) + if self.windowID != -1: + self.canvas.delete(self.windowID) + self.windowID = self.canvas.create_line(self.queueX + (col * 20) - 10, self.queueY - 13 + (row * 20), + self.queueX + (col * 20) - 10, self.queueY + 13 + (row * 20), width=2) + + def AddQueueEntry(self, block, index): + self.requestQueue.append((block, index)) + self.requestState.append(STATE_NULL) + if self.graphics: + (col, row) = self.QueueMap(index) + sizeHalf = self.queueBoxSize / 2.0 + (cx, cy) = (self.queueX + (col * self.queueBoxSize), self.queueY + (row * self.queueBoxSize)) + self.queueBoxID[index] = self.canvas.create_rectangle(cx - sizeHalf, cy - sizeHalf, cx + sizeHalf, cy + sizeHalf, fill='white') + self.queueTxtID[index] = self.canvas.create_text(cx, cy, anchor='center', text=str(block)) + + def SwitchColors(self, c): + if self.graphics: + self.canvas.itemconfig(self.queueBoxID[self.currentIndex], fill=c) + self.canvas.itemconfig(self.targetID, fill=c) + + def SwitchState(self, newState): + self.state = newState + self.requestState[self.currentIndex] = newState + + def RadiallyCloseTo(self, a1, a2): + if a1 > a2: + v = a1 - a2 + else: + v = a2 - a1 + if v < self.rotateSpeed: + return True + return False + + def DoneWithTransfer(self): + angleOffset = self.blockAngleOffset[self.armTrack] + # if int(self.angle) == (self.blockToAngleMap[self.currentBlock] + angleOffset) % 360: + if self.RadiallyCloseTo(self.angle, float((self.blockToAngleMap[self.currentBlock] + angleOffset) % 360)): + # print 'END TRANSFER', self.angle, self.timer + self.SwitchState(STATE_DONE) + self.requestCount += 1 + return True + return False + + def DoneWithRotation(self): + angleOffset = self.blockAngleOffset[self.armTrack] + # XXX there is a weird bug in here + # print self.timer, 'ROTATE:: ', self.currentBlock, 'currangle: ', self.angle, ' - mapangle: ', self.blockToAngleMap[self.currentBlock] + # print ' angleOffset ', angleOffset + # print ' blockMap ', (self.blockToAngleMap[self.currentBlock] - angleOffset) % 360 + # print ' self.angle ', self.angle, int(self.angle) + # if int(self.angle) == (self.blockToAngleMap[self.currentBlock] - angleOffset) % 360: + if self.RadiallyCloseTo(self.angle, float((self.blockToAngleMap[self.currentBlock] - angleOffset) % 360)): + self.SwitchState(STATE_XFER) + # print ' --> DONE WITH ROTATION!', self.timer + return True + return False + + def PlanSeek(self, track): + self.seekBegin = self.timer + self.SwitchColors('orange') + self.SwitchState(STATE_SEEK) + if track == self.armTrack: + self.rotBegin = self.timer + self.SwitchColors('lightblue') + self.SwitchState(STATE_ROTATE) + return + self.armTarget = track + self.armTargetX1 = self.spindleX - self.tracks[track] - (self.trackWidth / 2.0) + if track >= self.armTrack: + self.armSpeed = self.armSpeedBase + else: + self.armSpeed = - self.armSpeedBase + + def DoneWithSeek(self): + # move the disk arm + self.armX1 += self.armSpeed + self.armX2 += self.armSpeed + self.headX1 += self.armSpeed + self.headX2 += self.armSpeed + # update it on screen + if self.graphics: + self.canvas.coords(self.armID, self.armX1, self.armY1, self.armX2, self.armY2) + self.canvas.coords(self.headID, self.headX1, self.headY1, self.headX2, self.headY2) + # check if done + if (self.armSpeed > 0.0 and self.armX1 >= self.armTargetX1) or (self.armSpeed < 0.0 and self.armX1 <= self.armTargetX1): + self.armTrack = self.armTarget + return True + return False + + def DoSATF(self, rList): + minBlock = -1 + minIndex = -1 + minEst = -1 + + # print '**** DoSATF ****', rList + for (block, index) in rList: + if self.requestState[index] == STATE_DONE: + continue + track = self.blockToTrackMap[block] + angle = self.blockToAngleMap[block] + # print 'track', track, 'angle', angle + + # estimate seek time + dist = int(math.fabs(self.armTrack - track)) + seekEst = (self.trackWidth / self.armSpeedBase) * dist + # print 'dist', dist + # print 'seekEst', seekEst + + # estimate rotate time + angleOffset = self.blockAngleOffset[track] + # print 'angleOffset', angleOffset + # print 'self.angle', self.angle + angleAtArrival = (self.angle + (seekEst * self.rotateSpeed)) + while angleAtArrival > 360.0: + angleAtArrival -= 360.0 + # print 'self.rotateSpeed', self.rotateSpeed + # print 'angleAtArrival', angleAtArrival + rotDist = ((angle - angleOffset) - angleAtArrival) + while rotDist > 360.0: + rotDist -= 360.0 + while rotDist < 0.0: + rotDist += 360.0 + rotEst = rotDist / self.rotateSpeed + # print 'rotEst', rotDist, self.rotateSpeed, ' -> ', rotEst + + # finally, transfer + xferEst = (angleOffset * 2.0) / self.rotateSpeed + + # print 'xferEst', xferEst + + totalEst = seekEst + rotEst + xferEst + # print 'totalEst', seekEst, rotEst, xferEst, ' -> ', totalEst + + # print '--> block:%d seek:%d rotate:%d xfer:%d est:%d' % (block, seekEst, rotEst, xferEst, totalEst) + + # should probably pick one on same track in case of a TIE + if minEst == -1 or totalEst < minEst: + minEst = totalEst + minBlock = block + minIndex = index + # END loop + + # when done + self.totalEst = minEst + assert(minBlock != -1) + assert(minIndex != -1) + return (minBlock, minIndex) + + # + # actually doesn't quite do SSTF + # just finds all the blocks on the nearest track + # (whatever that may be) and returns it as a list + # + def DoSSTF(self, rList): + minDist = MAXTRACKS + minBlock = -1 + trackList = [] # all the blocks on a track + + for (block, index) in rList: + if self.requestState[index] == STATE_DONE: + continue + track = self.blockToTrackMap[block] + dist = int(math.fabs(self.armTrack - track)) + if dist < minDist: + trackList = [] + trackList.append((block, index)) + minDist = dist + elif dist == minDist: + trackList.append((block, index)) + assert(trackList != []) + return trackList + + def UpdateWindow(self): + if self.fairWindow == -1 and self.currWindow > 0 and self.currWindow < len(self.requestQueue): + self.currWindow += 1 + if self.graphics: + self.DrawWindow() + + def GetWindow(self): + if self.currWindow <= -1: + return len(self.requestQueue) + else: + if self.fairWindow != -1: + if self.requestCount > 0 and (self.requestCount % self.fairWindow == 0): + self.currWindow = self.currWindow + self.fairWindow + if self.currWindow > len(self.requestQueue): + self.currWindow = len(self.requestQueue) + if self.graphics: + self.DrawWindow() + return self.currWindow + else: + return self.currWindow + + def GetNextIO(self): + # check if done: if so, print stats and end animation + if self.requestCount == len(self.requestQueue): + self.UpdateTime() + self.PrintStats() + self.doAnimate = False + self.isDone = True + return + + # do policy: should set currentBlock, + if self.policy == 'FIFO': + (self.currentBlock, self.currentIndex) = self.requestQueue[self.requestCount] + self.DoSATF(self.requestQueue[self.requestCount:self.requestCount+1]) + elif self.policy == 'SATF' or self.policy == 'BSATF': + (self.currentBlock, self.currentIndex) = self.DoSATF(self.requestQueue[0:self.GetWindow()]) + elif self.policy == 'SSTF': + # first, find all the blocks on a given track (given window constraints) + trackList = self.DoSSTF(self.requestQueue[0:self.GetWindow()]) + # then, do SATF on those blocks (otherwise, will not do them in obvious order) + (self.currentBlock, self.currentIndex) = self.DoSATF(trackList) + else: + print 'policy (%s) not implemented' % self.policy + sys.exit(1) + + # once best block is decided, go ahead and do the seek + self.PlanSeek(self.blockToTrackMap[self.currentBlock]) + + # add another block? + if len(self.lateRequests) > 0 and self.lateCount < len(self.lateRequests): + self.AddRequest(self.lateRequests[self.lateCount]) + self.lateCount += 1 + + def Animate(self): + if self.graphics == True and self.doAnimate == False: + self.root.after(20, self.Animate) + return + + # timer + self.timer += 1 + self.UpdateTime() + + # see which blocks are rotating on the disk + # print 'SELF ANGLE', self.angle + self.angle = self.angle + self.rotateSpeed + if self.angle >= 360.0: + self.angle = 0.0 + + # move the blocks + if self.graphics: + for (track, angle, name, cid) in self.blockInfoList: + distFromSpindle = self.tracks[track] + na = angle - self.angle + xc = self.spindleX + (distFromSpindle * math.cos(math.radians(na))) + yc = self.spindleY + (distFromSpindle * math.sin(math.radians(na))) + if self.graphics: + self.canvas.coords(cid, xc, yc) + if self.currentBlock == name: + sz = self.targetSize + self.canvas.coords(self.targetID, xc-sz, yc-sz, xc+sz, yc+sz) + + # move the arm OR wait for a rotational delay + if self.state == STATE_SEEK: + if self.DoneWithSeek(): + self.rotBegin = self.timer + self.SwitchState(STATE_ROTATE) + self.SwitchColors('lightblue') + if self.state == STATE_ROTATE: + # check for read (disk arm must be settled) + if self.DoneWithRotation(): + self.xferBegin = self.timer + self.SwitchState(STATE_XFER) + self.SwitchColors('green') + if self.state == STATE_XFER: + if self.DoneWithTransfer(): + self.DoRequestStats() + self.SwitchState(STATE_DONE) + self.SwitchColors('red') + self.UpdateWindow() + currentBlock = self.currentBlock + self.GetNextIO() + nextBlock = self.currentBlock + if self.blockToTrackMap[currentBlock] == self.blockToTrackMap[nextBlock]: + if (currentBlock == self.tracksBeginEnd[self.armTrack][1] and nextBlock == self.tracksBeginEnd[self.armTrack][0]) or (currentBlock + 1 == nextBlock): + # need a special case here: to handle when we stay in transfer mode + (self.rotBegin, self.seekBegin, self.xferBegin) = (self.timer, self.timer, self.timer) + self.SwitchState(STATE_XFER) + self.SwitchColors('green') + + + + # make sure to keep the animation going! + if self.graphics: + self.root.after(20, self.Animate) + + def DoRequestStats(self): + seekTime = self.rotBegin - self.seekBegin + rotTime = self.xferBegin - self.rotBegin + xferTime = self.timer - self.xferBegin + totalTime = self.timer - self.seekBegin + + if self.compute == True: + print 'Block: %3d Seek:%3d Rotate:%3d Transfer:%3d Total:%4d' % (self.currentBlock, seekTime, rotTime, xferTime, totalTime) + + # if int(totalTime) != int(self.totalEst): + # print 'INTERNAL ERROR: estimate was', self.totalEst, 'whereas actual time to access block was', totalTime + # print 'Please report this bug and as much information as possible so as to make it easy to recreate. Thanks!' + + # update stats + self.seekTotal += seekTime + self.rotTotal += rotTime + self.xferTotal += xferTime + + + + def PrintStats(self): + if self.compute == True: + print '\nTOTALS Seek:%3d Rotate:%3d Transfer:%3d Total:%4d\n' % (self.seekTotal, self.rotTotal, self.xferTotal, self.timer) + +# END: class Disk + + + +# +# MAIN SIMULATOR +# +parser = OptionParser() +parser.add_option('-s', '--seed', default='0', help='Random seed', action='store', type='int', dest='seed') +parser.add_option('-a', '--addr', default='-1', help='Request list (comma-separated) [-1 -> use addrDesc]', action='store', type='string', dest='addr') +parser.add_option('-A', '--addrDesc', default='5,-1,0', help='Num requests, max request (-1->all), min request', action='store', type='string', dest='addrDesc') +parser.add_option('-S', '--seekSpeed', default='1', help='Speed of seek', action='store', type='string', dest='seekSpeed') +parser.add_option('-R', '--rotSpeed', default='1', help='Speed of rotation', action='store', type='string', dest='rotateSpeed') +parser.add_option('-p', '--policy', default='FIFO', help='Scheduling policy (FIFO, SSTF, SATF, BSATF)', action='store', type='string', dest='policy') +parser.add_option('-w', '--schedWindow', default=-1, help='Size of scheduling window (-1 -> all)', action='store', type='int', dest='window') +parser.add_option('-o', '--skewOffset', default=0, help='Amount of skew (in blocks)', action='store', type='int', dest='skew') +parser.add_option('-z', '--zoning', default='30,30,30', help='Angles between blocks on outer,middle,inner tracks', action='store', type='string', dest='zoning') +parser.add_option('-G', '--graphics', default=False, help='Turn on graphics', action='store_true', dest='graphics') +parser.add_option('-l', '--lateAddr', default='-1', help='Late: request list (comma-separated) [-1 -> random]', action='store', type='string', dest='lateAddr') +parser.add_option('-L', '--lateAddrDesc', default='0,-1,0', help='Num requests, max request (-1->all), min request', action='store', type='string', dest='lateAddrDesc') +parser.add_option('-c', '--compute', default=False, help='Compute the answers', action='store_true', dest='compute') +(options, args) = parser.parse_args() + +print 'OPTIONS seed', options.seed +print 'OPTIONS addr', options.addr +print 'OPTIONS addrDesc', options.addrDesc +print 'OPTIONS seekSpeed', options.seekSpeed +print 'OPTIONS rotateSpeed', options.rotateSpeed +print 'OPTIONS skew', options.skew +print 'OPTIONS window', options.window +print 'OPTIONS policy', options.policy +print 'OPTIONS compute', options.compute +print 'OPTIONS graphics', options.graphics +print 'OPTIONS zoning', options.zoning +print 'OPTIONS lateAddr', options.lateAddr +print 'OPTIONS lateAddrDesc', options.lateAddrDesc +print '' + +if options.window == 0: + print 'Scheduling window (%d) must be positive or -1 (which means a full window)' % options.window + sys.exit(1) + +if options.graphics and options.compute == False: + print '\nWARNING: Setting compute flag to True, as graphics are on\n' + options.compute = True + +# set up simulator info +d = Disk(addr=options.addr, addrDesc=options.addrDesc, lateAddr=options.lateAddr, lateAddrDesc=options.lateAddrDesc, + policy=options.policy, seekSpeed=float(options.seekSpeed), rotateSpeed=float(options.rotateSpeed), + skew=options.skew, window=options.window, compute=options.compute, graphics=options.graphics, zoning=options.zoning) + +# run simulation +d.Go() diff --git a/related_info/ostep/ostep16-raid.md b/related_info/ostep/ostep16-raid.md new file mode 100644 index 0000000..0981a0b --- /dev/null +++ b/related_info/ostep/ostep16-raid.md @@ -0,0 +1,223 @@ + +This section introduces raid.py, a simple RAID simulator you can use to shore +up your knowledge of how RAID systems work. It has a number of options, as we +see below: + +Usage: raid.py [options] + +Options: + -h, --help show this help message and exit + -s SEED, --seed=SEED the random seed + -D NUMDISKS, --numDisks=NUMDISKS + number of disks in RAID + -C CHUNKSIZE, --chunkSize=CHUNKSIZE + chunk size of the RAID + -n NUMREQUESTS, --numRequests=NUMREQUESTS + number of requests to simulate + -S SIZE, --reqSize=SIZE + size of requests + -W WORKLOAD, --workload=WORKLOAD + either "rand" or "seq" workloads + -w WRITEFRAC, --writeFrac=WRITEFRAC + write fraction (100->all writes, 0->all reads) + -R RANGE, --randRange=RANGE + range of requests (when using "rand" workload) + -L LEVEL, --level=LEVEL + RAID level (0, 1, 4, 5) + -5 RAID5TYPE, --raid5=RAID5TYPE + RAID-5 left-symmetric "LS" or left-asym "LA" + -r, --reverse instead of showing logical ops, show physical + -t, --timing use timing mode, instead of mapping mode + -c, --compute compute answers for me + +In its basic mode, you can use it to understand how the different RAID levels +map logical blocks to underlying disks and offsets. For example, let's say we +wish to see how a simple striping RAID (RAID-0) with four disks does this +mapping. + +prompt> ./raid.py -n 5 -L 0 -R 20 +... +LOGICAL READ from addr:16 size:4096 + Physical reads/writes? + +LOGICAL READ from addr:8 size:4096 + Physical reads/writes? + +LOGICAL READ from addr:10 size:4096 + Physical reads/writes? + +LOGICAL READ from addr:15 size:4096 + Physical reads/writes? + +LOGICAL READ from addr:9 size:4096 + Physical reads/writes? + +In this example, we simulate five requests (-n 5), specifying RAID level zero +(-L 0), and restrict the range of random requests to just the first twenty +blocks of the RAID (-R 20). The result is a series of random reads to the +first twenty blocks of the RAID; the simulator then asks you to guess which +underlying disks/offsets were accessed to service the request, for each +logical read. + +In this case, calculating the answers is easy: in RAID-0, recall that the +underlying disk and offset that services a request is calculated via modulo +arithmetic: + + disk = address % number_of_disks + offset = address / number_of_disks + +Thus, the first request to 16 should be serviced by disk 0, at offset 4. And +so forth. You can, as usual see the answers (once you've computed them!), by +using the handy "-c" flag to compute the results. + +prompt> ./raid.py -R 20 -n 5 -L 0 -c +... +LOGICAL READ from addr:16 size:4096 + read [disk 0, offset 4] + +LOGICAL READ from addr:8 size:4096 + read [disk 0, offset 2] + +LOGICAL READ from addr:10 size:4096 + read [disk 2, offset 2] + +LOGICAL READ from addr:15 size:4096 + read [disk 3, offset 3] + +LOGICAL READ from addr:9 size:4096 + read [disk 1, offset 2] + + +Because we like to have fun, you can also do this problem in reverse, with the +"-r" flag. Running the simulator this way shows you the low-level disk reads +and writes, and asks you to reverse engineer which logical request must have +been given to the RAID: + +prompt> ./raid.py -R 20 -n 5 -L 0 -r +... +LOGICAL OPERATION is ? + read [disk 0, offset 4] + +LOGICAL OPERATION is ? + read [disk 0, offset 2] + +LOGICAL OPERATION is ? + read [disk 2, offset 2] + +LOGICAL OPERATION is ? + read [disk 3, offset 3] + +LOGICAL OPERATION is ? + read [disk 1, offset 2] + +You can again use -c to show the answers. To get more variety, a +different random seed (-s) can be given. + +Even further variety is available by examining different RAID levels. +In the simulator, RAID-0 (block striping), RAID-1 (mirroring), RAID-4 +(block-striping plus a single parity disk), and RAID-5 (block-striping with +rotating parity) are supported. + +In this next example, we show how to run the simulator in mirrored mode. We +show the answers to save space: + +prompt> ./raid.py -R 20 -n 5 -L 1 -c +... +LOGICAL READ from addr:16 size:4096 + read [disk 0, offset 8] + +LOGICAL READ from addr:8 size:4096 + read [disk 0, offset 4] + +LOGICAL READ from addr:10 size:4096 + read [disk 1, offset 5] + +LOGICAL READ from addr:15 size:4096 + read [disk 3, offset 7] + +LOGICAL READ from addr:9 size:4096 + read [disk 2, offset 4] + +You might notice a few things about this example. First, the mirrored RAID-1 +assumes a striped layout (which some might call RAID-01), where logical block +0 is mapped to the 0th block of disks 0 and 1, logical block 1 is mapped to +the 0th blocks of disks 2 and 3, and so forth (in this four-disk example). +Second, when reading a single block from a mirrored RAID system, the RAID has +a choice of which of two blocks to read. In this simulator, we use a +relatively silly way: for even-numbered logical blocks, the RAID chooses the +even-numbered disk in the pair; the odd disk is used for odd-numbered logical +blocks. This is done to make the results of each run easy to guess for you +(instead of, for example, a random choice). + +We can also explore how writes behave (instead of just reads) with the -w +flag, which specifies the "write fraction" of a workload, i.e., the fraction +of requests that are writes. By default, it is set to zero, and thus the +examples so far were 100\% reads. Let's see what happens to our mirrored RAID +when some writes are introduced: + +prompt> ./raid.py -R 20 -n 5 -L 1 -w 100 -c +... +LOGICAL WRITE to addr:16 size:4096 + write [disk 0, offset 8] write [disk 1, offset 8] + +LOGICAL WRITE to addr:8 size:4096 + write [disk 0, offset 4] write [disk 1, offset 4] + +LOGICAL WRITE to addr:10 size:4096 + write [disk 0, offset 5] write [disk 1, offset 5] + +LOGICAL WRITE to addr:15 size:4096 + write [disk 2, offset 7] write [disk 3, offset 7] + +LOGICAL WRITE to addr:9 size:4096 + write [disk 2, offset 4] write [disk 3, offset 4] + +With writes, instead of generating just a single low-level disk operation, the +RAID must of course update both disks, and hence two writes are issued. +Even more interesting things happen with RAID-4 and RAID-5, as you might +guess; we'll leave the exploration of such things to you in the questions +below. + +The remaining options are discovered via the help flag. They are: + +Options: + -h, --help show this help message and exit + -s SEED, --seed=SEED the random seed + -D NUMDISKS, --numDisks=NUMDISKS + number of disks in RAID + -C CHUNKSIZE, --chunkSize=CHUNKSIZE + chunk size of the RAID + -n NUMREQUESTS, --numRequests=NUMREQUESTS + number of requests to simulate + -S SIZE, --reqSize=SIZE + size of requests + -W WORKLOAD, --workload=WORKLOAD + either "rand" or "seq" workloads + -w WRITEFRAC, --writeFrac=WRITEFRAC + write fraction (100->all writes, 0->all reads) + -R RANGE, --randRange=RANGE + range of requests (when using "rand" workload) + -L LEVEL, --level=LEVEL + RAID level (0, 1, 4, 5) + -5 RAID5TYPE, --raid5=RAID5TYPE + RAID-5 left-symmetric "LS" or left-asym "LA" + -r, --reverse instead of showing logical ops, show physical + -t, --timing use timing mode, instead of mapping mode + -c, --compute compute answers for me + + +The -C flag allows you to set the chunk size of the RAID, instead of using the +default size of one 4-KB block per chunk. The size of each request can be +similarly adjusted with the -S flag. The default workload accesses random +blocks; use -W sequential to explore the behavior of sequential accesses. With +RAID-5, two different layout schemes are available, left-symmetric and +left-asymmetric; use -5 LS or -5 LA to try those out with RAID-5 (-L 5). + +Finally, in timing mode (-t), the simulator uses an incredibly simple disk +model to estimate how long a set of requests takes, instead of just focusing +on mappings. In this mode, a random request takes 10 milliseconds, whereas a +sequential request takes 0.1 milliseconds. The disk is assumed to have a tiny +number of blocks per track (100), and a similarly small number of tracks +(100). You can thus use the simulator to estimate RAID performance under some +different workloads. + diff --git a/related_info/ostep/ostep16-raid.py b/related_info/ostep/ostep16-raid.py new file mode 100755 index 0000000..4fa1a89 --- /dev/null +++ b/related_info/ostep/ostep16-raid.py @@ -0,0 +1,447 @@ +#! /usr/bin/env python + +import math +import random +from optparse import OptionParser + +# minimum unit of transfer to RAID +BLOCKSIZE = 4096 + +def convert(size): + length = len(size) + lastchar = size[length-1] + if (lastchar == 'k') or (lastchar == 'K'): + m = 1024 + nsize = int(size[0:length-1]) * m + elif (lastchar == 'm') or (lastchar == 'M'): + m = 1024*1024 + nsize = int(size[0:length-1]) * m + elif (lastchar == 'g') or (lastchar == 'G'): + m = 1024*1024*1024 + nsize = int(size[0:length-1]) * m + else: + nsize = int(size) + return nsize + +class disk: + def __init__(self, seekTime=10, xferTime=0.1, queueLen=8): + # these are both in milliseconds + # seek is the time to seek (simple constant amount) + # transfer is the time to read one block + self.seekTime = seekTime + self.xferTime = xferTime + + # length of scheduling queue + self.queueLen = queueLen + + # current location: make it negative so that whatever + # the first read is, it causes a seek + self.currAddr = -10000 + + # queue + self.queue = [] + + # disk geometry + self.numTracks = 100 + self.blocksPerTrack = 100 + self.blocksPerDisk = self.numTracks * self.blocksPerTrack + + # stats + self.countIO = 0 + self.countSeq = 0 + self.countNseq = 0 + self.countRand = 0 + self.utilTime = 0 + + def stats(self): + return (self.countIO, self.countSeq, self.countNseq, self.countRand, self.utilTime) + + def enqueue(self, addr): + assert(addr < self.blocksPerDisk) + self.countIO += 1 + + # check if this is on the same track, or a different one + currTrack = self.currAddr / self.numTracks + newTrack = addr / self.numTracks + + # absolute diff + diff = addr - self.currAddr + + # if on the same track... + if currTrack == newTrack or diff < self.blocksPerTrack: + if diff == 1: + self.countSeq += 1 + else: + self.countNseq += 1 + self.utilTime += (diff * self.xferTime) + else: + self.countRand += 1 + self.utilTime += (self.seekTime + self.xferTime) + self.currAddr = addr + + def go(self): + return self.utilTime + +class raid: + def __init__(self, chunkSize='4k', numDisks=4, level=0, timing=False, reverse=False, solve=False, raid5type='LS'): + chunkSize = int(convert(chunkSize)) + self.chunkSize = chunkSize / BLOCKSIZE + self.numDisks = numDisks + self.raidLevel = level + self.timing = timing + self.reverse = reverse + self.solve = solve + self.raid5type = raid5type + + if (chunkSize % BLOCKSIZE) != 0: + print 'chunksize (%d) must be multiple of blocksize (%d): %d' % (chunkSize, BLOCKSIZE, self.chunkSize % BLOCKSIZE) + exit(1) + if self.raidLevel == 1 and numDisks % 2 != 0: + print 'raid1: disks (%d) must be a multiple of two' % numDisks + exit(1) + + if self.raidLevel == 4: + self.blocksInStripe = (self.numDisks - 1) * self.chunkSize + self.pdisk = self.numDisks - 1 + if self.raidLevel == 5: + self.blocksInStripe = (self.numDisks - 1) * self.chunkSize + self.pdisk = -1 + + self.disks = [] + for i in range(self.numDisks): + self.disks.append(disk()) + + # print per-disk stats + def stats(self, totalTime): + for d in range(self.numDisks): + s = self.disks[d].stats() + if s[4] == totalTime: + print 'disk:%d busy: %.2f I/Os: %5d (sequential:%d nearly:%d random:%d)' % (d, (100.0*float(s[4])/totalTime), s[0], s[1], s[2], s[3]) + elif s[4] == 0: + print 'disk:%d busy: %.2f I/Os: %5d (sequential:%d nearly:%d random:%d)' % (d, (100.0*float(s[4])/totalTime), s[0], s[1], s[2], s[3]) + else: + print 'disk:%d busy: %.2f I/Os: %5d (sequential:%d nearly:%d random:%d)' % (d, (100.0*float(s[4])/totalTime), s[0], s[1], s[2], s[3]) + + # global enqueue function + def enqueue(self, addr, size, isWrite): + # should we print out the logical operation? + if self.timing == False: + if self.solve or self.reverse==False: + if isWrite: + print 'LOGICAL WRITE to addr:%d size:%d' % (addr, size * BLOCKSIZE) + else: + print 'LOGICAL READ from addr:%d size:%d' % (addr, size * BLOCKSIZE) + if self.solve == False: + print ' Physical reads/writes?\n' + else: + print 'LOGICAL OPERATION is ?' + + # should we print out the physical operations? + if self.timing == False and (self.solve or self.reverse==True): + self.printPhysical = True + else: + self.printPhysical = False + + if self.raidLevel == 0: + self.enqueue0(addr, size, isWrite) + elif self.raidLevel == 1: + self.enqueue1(addr, size, isWrite) + elif self.raidLevel == 4 or self.raidLevel == 5: + self.enqueue45(addr, size, isWrite) + + # process disk workloads one at a time, returning final completion time + def go(self): + tmax = 0 + for d in range(self.numDisks): + # print '**** disk ****', d + t = self.disks[d].go() + if t > tmax: + tmax = t + return tmax + + # helper functions + def doSingleRead(self, disk, off, doNewline=False): + if self.printPhysical: + print ' read [disk %d, offset %d] ' % (disk, off), + if doNewline: + print '' + self.disks[disk].enqueue(off) + + def doSingleWrite(self, disk, off, doNewline=False): + if self.printPhysical: + print ' write [disk %d, offset %d] ' % (disk, off), + if doNewline: + print '' + self.disks[disk].enqueue(off) + + # + # mapping for RAID 0 (striping) + # + def bmap0(self, bnum): + cnum = bnum / self.chunkSize + coff = bnum % self.chunkSize + return (cnum % self.numDisks, (cnum / self.numDisks) * self.chunkSize + coff) + + def enqueue0(self, addr, size, isWrite): + # can ignore isWrite, as I/O pattern is the same for striping + for b in range(addr, addr+size): + (disk, off) = self.bmap0(b) + if isWrite: + self.doSingleWrite(disk, off, True) + else: + self.doSingleRead(disk, off, True) + if self.timing == False and self.printPhysical: + print '' + + # + # mapping for RAID 1 (mirroring) + # + def bmap1(self, bnum): + cnum = bnum / self.chunkSize + coff = bnum % self.chunkSize + disk = 2 * (cnum % (self.numDisks / 2)) + return (disk, disk + 1, (cnum / (self.numDisks / 2)) * self.chunkSize + coff) + + def enqueue1(self, addr, size, isWrite): + for b in range(addr, addr+size): + (disk1, disk2, off) = self.bmap1(b) + # print 'enqueue:', addr, size, '-->', m + if isWrite: + self.doSingleWrite(disk1, off, False) + self.doSingleWrite(disk2, off, True) + else: + # the raid-1 read balancing algorithm is here; + # could be something more intelligent -- + # instead, it is just based on the disk offset + # to produce something easily reproducible + if off % 2 == 0: + self.doSingleRead(disk1, off, True) + else: + self.doSingleRead(disk2, off, True) + if self.timing == False and self.printPhysical: + print '' + + # + # mapping for RAID 4 (parity disk) + # + # assumes (for now) that there is just one parity disk + # + def bmap4(self, bnum): + cnum = bnum / self.chunkSize + coff = bnum % self.chunkSize + return (cnum % (self.numDisks - 1), (cnum / (self.numDisks - 1)) * self.chunkSize + coff) + + def pmap4(self, snum): + return self.pdisk + + # + # mapping for RAID 5 (rotated parity) + # + def __bmap5(self, bnum): + cnum = bnum / self.chunkSize + coff = bnum % self.chunkSize + ddsk = cnum / (self.numDisks - 1) + doff = (ddsk * self.chunkSize) + coff + disk = cnum % (self.numDisks - 1) + col = (ddsk % self.numDisks) + pdsk = (self.numDisks - 1) - col + + # supports left-asymmetric and left-symmetric layouts + if self.raid5type == 'LA': + if disk >= pdisk: + disk += 1 + elif self.raid5type == 'LS': + disk = (disk - col) % (self.numDisks) + else: + print 'error: no such RAID scheme' + exit(1) + assert(disk != pdsk) + return (disk, pdsk, doff) + + # yes this is lame (redundant call to __bmap5 is serious programmer laziness) + def bmap5(self, bnum): + (disk, pdisk, off) = self.__bmap5(bnum) + return (disk, off) + + # this too is lame (redundant call to __bmap5 is serious programmer laziness) + def pmap5(self, snum): + (disk, pdisk, off) = self.__bmap5(snum * self.blocksInStripe) + return pdisk + + # RAID 4/5 helper routine to write out some blocks in a stripe + def doPartialWrite(self, stripe, begin, end, bmap, pmap): + numWrites = end - begin + pdisk = pmap(stripe) + if (numWrites + 1) <= (self.blocksInStripe - numWrites): + # SUBTRACTIVE PARITY + # print 'SUBTRACTIVE' + offList = [] + for voff in range(begin, end): + (disk, off) = bmap(voff) + self.doSingleRead(disk, off) + if off not in offList: + offList.append(off) + for i in range(len(offList)): + self.doSingleRead(pdisk, offList[i], i == (len(offList) - 1)) + else: + # ADDITIVE PARITY + # print 'ADDITIVE' + stripeBegin = stripe * self.blocksInStripe + stripeEnd = stripeBegin + self.blocksInStripe + for voff in range(stripeBegin, begin): + (disk, off) = bmap(voff) + self.doSingleRead(disk, off, (voff == (begin - 1)) and (end == stripeEnd)) + for voff in range(end, stripeEnd): + (disk, off) = bmap(voff) + self.doSingleRead(disk, off, voff == (stripeEnd - 1)) + + # WRITES: same for additive or subtractive parity + offList = [] + for voff in range(begin, end): + (disk, off) = bmap(voff) + self.doSingleWrite(disk, off) + if off not in offList: + offList.append(off) + for i in range(len(offList)): + self.doSingleWrite(pdisk, offList[i], i == (len(offList) - 1)) + + # RAID 4/5 enqueue routine + def enqueue45(self, addr, size, isWrite): + if self.raidLevel == 4: + (bmap, pmap) = (self.bmap4, self.pmap4) + elif self.raidLevel == 5: + (bmap, pmap) = (self.bmap5, self.pmap5) + + if isWrite == False: + for b in range(addr, addr+size): + (disk, off) = bmap(b) + self.doSingleRead(disk, off) + else: + # process the write request, one stripe at a time + initStripe = (addr) / self.blocksInStripe + finalStripe = (addr + size - 1) / self.blocksInStripe + + left = size + begin = addr + for stripe in range(initStripe, finalStripe + 1): + endOfStripe = (stripe * self.blocksInStripe) + self.blocksInStripe + + if left >= self.blocksInStripe: + end = begin + self.blocksInStripe + else: + end = begin + left + + if end >= endOfStripe: + end = endOfStripe + + self.doPartialWrite(stripe, begin, end, bmap, pmap) + + left -= (end - begin) + begin = end + + # for all cases, print this for pretty-ness in mapping mode + if self.timing == False and self.printPhysical: + print '' + +# +# main program +# +parser = OptionParser() + +parser.add_option('-s', '--seed', default=0, help='the random seed', action='store', type='int', dest='seed') +parser.add_option('-D', '--numDisks', default=4, help='number of disks in RAID', action='store', type='int', dest='numDisks') +parser.add_option('-C', '--chunkSize', default='4k', help='chunk size of the RAID', action='store', type='string', dest='chunkSize') +parser.add_option('-n', '--numRequests', default=10, help='number of requests to simulate', action='store', type='int', dest='numRequests') +parser.add_option('-S', '--reqSize', default='4k', help='size of requests', action='store', type='string', dest='size') +parser.add_option('-W', '--workload', default='rand', help='either "rand" or "seq" workloads', action='store', type='string', dest='workload') +parser.add_option('-w', '--writeFrac', default=0, help='write fraction (100->all writes, 0->all reads)', action='store', type='int', dest='writeFrac') +parser.add_option('-R', '--randRange', default=10000, help='range of requests (when using "rand" workload)', action='store', type='int', dest='range') +parser.add_option('-L', '--level', default=0, help='RAID level (0, 1, 4, 5)', action='store', type='int', dest='level') +parser.add_option('-5', '--raid5', default='LS', help='RAID-5 left-symmetric "LS" or left-asym "LA"', action='store', type='string', dest='raid5type') +parser.add_option('-r', '--reverse', default=False, help='instead of showing logical ops, show physical', action='store_true', dest='reverse') +parser.add_option('-t', '--timing', default=False, help='use timing mode, instead of mapping mode', action='store_true', dest='timing') +parser.add_option('-c', '--compute', default=False, help='compute answers for me', action='store_true', dest='solve') + +(options, args) = parser.parse_args() + +print 'ARG blockSize', BLOCKSIZE +print 'ARG seed', options.seed +print 'ARG numDisks', options.numDisks +print 'ARG chunkSize', options.chunkSize +print 'ARG numRequests', options.numRequests +print 'ARG reqSize', options.size +print 'ARG workload', options.workload +print 'ARG writeFrac', options.writeFrac +print 'ARG randRange', options.range +print 'ARG level', options.level +print 'ARG raid5', options.raid5type +print 'ARG reverse', options.reverse +print 'ARG timing', options.timing + +print '' + +writeFrac = float(options.writeFrac) / 100.0 +assert(writeFrac >= 0.0 and writeFrac <= 1.0) + +random.seed(options.seed) + +size = convert(options.size) +if size % BLOCKSIZE != 0: + print 'error: request size (%d) must be a multiple of BLOCKSIZE (%d)' % (size, BLOCKSIZE) + exit(1) +size = size / BLOCKSIZE + +if options.workload == 'seq' or options.workload == 's' or options.workload == 'sequential': + workloadIsSequential = True +elif options.workload == 'rand' or options.workload == 'r' or options.workload == 'random': + workloadIsSequential = False +else: + print 'error: workload must be either r/rand/random or s/seq/sequential' + exit(1) + +assert(options.level == 0 or options.level == 1 or options.level == 4 or options.level == 5) +if options.level != 0 and options.numDisks < 2: + print 'RAID-4 and RAID-5 need more than 1 disk' + exit(1) + +if options.level == 5 and options.raid5type != 'LA' and options.raid5type != 'LS': + print 'Only two types of RAID-5 supported: left-asymmetric (LA) and left-symmetric (LS) (%s is not)' % options.raid5type + exit(1) + +# instantiate RAID +r = raid(chunkSize=options.chunkSize, numDisks=options.numDisks, level=options.level, timing=options.timing, + reverse=options.reverse, solve=options.solve, raid5type=options.raid5type) + +# generate requests +off = 0 +for i in range(options.numRequests): + if workloadIsSequential == True: + blk = off + off += size + else: + blk = int(random.random() * options.range) + if random.random() < writeFrac: + r.enqueue(blk, size, True) + else: + r.enqueue(blk, size, False) + +# process requests +t = r.go() + +# print out some final info, if needed +if options.timing == False: + print '' + exit(0) + +if options.solve: + print '' + r.stats(t) + print '' + print 'STAT totalTime', t + print '' +else: + print '' + print 'Estimate how long the workload should take to complete.' + print '- Roughly how many requests should each disk receive?' + print '- How many requests are random, how many sequential?' + print '' diff --git a/related_info/ostep/ostep2-segmentation.md b/related_info/ostep/ostep2-segmentation.md new file mode 100644 index 0000000..b75393e --- /dev/null +++ b/related_info/ostep/ostep2-segmentation.md @@ -0,0 +1,166 @@ + +This program allows you to see how address translations are performed in a +system with segmentation. The segmentation that this system uses is pretty +simple: an address space has just *two* segments; further, the top bit of the +virtual address generated by the process determines which segment the address +is in: 0 for segment 0 (where, say, code and the heap would reside) and 1 for +segment 1 (where the stack lives). Segment 0 grows in a positive direction +(towards higher addresses), whereas segment 1 grows in the negative direction. + +Visually, the address space looks like this: + + --------------- virtual address 0 + | seg0 | + | | + | | + |-------------| + | | + | | + | | + | | + |(unallocated)| + | | + | | + | | + |-------------| + | | + | seg1 | + |-------------| virtual address max (size of address space) + +With segmentation, as you might recall, there is a base/limit pair of +registers per segment. Thus, in this problem, there are two base/limit +pairs. The segment-0 base tells which physical address the *top* of segment 0 +has been placed in physical memory and the limit tells how big the segment is; +the segment-1 base tells where the *bottom* of segment 1 has been placed in +physical memory and the corresponding limit also tells us how big the segment +is (or how far it grows in the negative direction). + +As before, there are two steps to running the program to test out your +understanding of segmentation. First, run without the "-c" flag to generate a +set of translations and see if you can correctly perform the address +translations yourself. Then, when done, run with the "-c" flag to check your +answers. + +For example, to run with the default flags, type: + +prompt> ./segmentation.py + +(or +prompt> python ./segmentation.py +if that doesn't work) + +You should see this: + ARG seed 0 + ARG address space size 1k + ARG phys mem size 16k + + Segment register information: + + Segment 0 base (grows positive) : 0x00001aea (decimal 6890) + Segment 0 limit : 472 + + Segment 1 base (grows negative) : 0x00001254 (decimal 4692) + Segment 1 limit : 450 + + Virtual Address Trace + VA 0: 0x0000020b (decimal: 523) --> PA or segmentation violation? + VA 1: 0x0000019e (decimal: 414) --> PA or segmentation violation? + VA 2: 0x00000322 (decimal: 802) --> PA or segmentation violation? + VA 3: 0x00000136 (decimal: 310) --> PA or segmentation violation? + VA 4: 0x000001e8 (decimal: 488) --> PA or segmentation violation? + + For each virtual address, either write down the physical address it translates + to OR write down that it is an out-of-bounds address (a segmentation + violation). For this problem, you should assume a simple address space with + two segments: the top bit of the virtual address can thus be used to check + whether the virtual address is in segment 0 (topbit=0) or segment 1 + (topbit=1). Note that the base/limit pairs given to you grow in different + directions, depending on the segment, i.e., segment 0 grows in the positive + direction, whereas segment 1 in the negative. + +Then, after you have computed the translations in the virtual address trace, +run the program again with the "-c" flag. You will see the following (not +including the redundant information): + + Virtual Address Trace + VA 0: 0x0000020b (decimal: 523) --> SEGMENTATION VIOLATION (SEG1) + VA 1: 0x0000019e (decimal: 414) --> VALID in SEG0: 0x00001c88 (decimal: 7304) + VA 2: 0x00000322 (decimal: 802) --> VALID in SEG1: 0x00001176 (decimal: 4470) + VA 3: 0x00000136 (decimal: 310) --> VALID in SEG0: 0x00001c20 (decimal: 7200) + VA 4: 0x000001e8 (decimal: 488) --> SEGMENTATION VIOLATION (SEG0) + +As you can see, with -c, the program translates the addresses for you, and +hence you can check if you understand how a system using segmentation +translates addresses. + +Of course, there are some parameters you can use to give yourself different +problems. One particularly important parameter is the -s or -seed parameter, +which lets you generate different problems by passing in a different random +seed. Of course, make sure to use the same random seed when you are generating +a problem and then solving it. + +There are also some parameters you can use to play with different-sized +address spaces and physical memories. For example, to experiment with +segmentation in a tiny system, you might type: + + prompt> ./segmentation.py -s 100 -a 16 -p 32 + ARG seed 0 + ARG address space size 16 + ARG phys mem size 32 + + Segment register information: + + Segment 0 base (grows positive) : 0x00000018 (decimal 24) + Segment 0 limit : 4 + + Segment 1 base (grows negative) : 0x00000012 (decimal 18) + Segment 1 limit : 5 + + Virtual Address Trace + VA 0: 0x0000000c (decimal: 12) --> PA or segmentation violation? + VA 1: 0x00000008 (decimal: 8) --> PA or segmentation violation? + VA 2: 0x00000001 (decimal: 1) --> PA or segmentation violation? + VA 3: 0x00000007 (decimal: 7) --> PA or segmentation violation? + VA 4: 0x00000000 (decimal: 0) --> PA or segmentation violation? + +which tells the program to generate virtual addresses for a 16-byte address +space placed somewhere in a 32-byte physical memory. As you can see, the +resulting virtual addresses are tiny (12, 8, 1, 7, and 0). As you can also +see, the program picks tiny base register and limit values, as +appropriate. Run with -c to see the answers. + +This example should also show you exactly what each base pair means. For +example, segment 0's base is set to a physical address of 24 (decimal) and is +of size 4 bytes. Thus, *virtual* addresses 0, 1, 2, and 3 are in segment 0 and +valid, and map to physical addresses 24, 25, 26, and 27, respectively. + +Slightly more tricky is the negative-direction-growing segment 1. In the tiny +example above, segment 1's base register is set to physical address 18, with a +size of 5 bytes. That means that the *last* five bytes of the virtual address +space, in this case 11, 12, 13, 14, and 15, are valid virtual addresses, and +that they map to physical addresses 13, 14, 15, 16, and 17, respectively. + +If that doesn't make sense, read it again -- you will have to make sense of +how this works in order to do any of these problems. + +Note you can specify bigger values by tacking a "k", "m", or even "g" onto the +values you pass in with the -a or -p flags, as in "kilobytes", "megabytes", +and "gigabytes". Thus, if you wanted to do some translations with a 1-MB +address space set in a 32-MB physical memory, you might type: + +prompt> ./segmentation.py -a 1m -p 32m + +If you want to get even more specific, you can set the base register and limit +register values yourself, with the --b0, --l0, --b1, and --l1 registers. Try +them and see. + +Finally, you can always run + +prompt> ./segmentation.py -h + +to get a complete list of flags and options. + +Enjoy! + + + diff --git a/related_info/ostep/ostep2-segmentation.py b/related_info/ostep/ostep2-segmentation.py new file mode 100755 index 0000000..d6bbd5b --- /dev/null +++ b/related_info/ostep/ostep2-segmentation.py @@ -0,0 +1,193 @@ +#! /usr/bin/env python + +import sys +from optparse import OptionParser +import random +import math + +def convert(size): + length = len(size) + lastchar = size[length-1] + if (lastchar == 'k') or (lastchar == 'K'): + m = 1024 + nsize = int(size[0:length-1]) * m + elif (lastchar == 'm') or (lastchar == 'M'): + m = 1024*1024 + nsize = int(size[0:length-1]) * m + elif (lastchar == 'g') or (lastchar == 'G'): + m = 1024*1024*1024 + nsize = int(size[0:length-1]) * m + else: + nsize = int(size) + return nsize + + +# +# main program +# +parser = OptionParser() +parser.add_option("-s", "--seed", default=0, help="the random seed", + action="store", type="int", dest="seed") +parser.add_option("-A", "--addresses", default="-1", + help="a set of comma-separated pages to access; -1 means randomly generate", + action="store", type="string", dest="addresses") +parser.add_option("-a", "--asize", default="1k", + help="address space size (e.g., 16, 64k, 32m, 1g)", + action="store", type="string", dest="asize") +parser.add_option("-p", "--physmem", default="16k", + help="physical memory size (e.g., 16, 64k, 32m, 1g)", + action="store", type="string", dest="psize") +parser.add_option("-n", "--numaddrs", default=5, + help="number of virtual addresses to generate", + action="store", type="int", dest="num") +parser.add_option("-b", "--b0", default="-1", + help="value of segment 0 base register", + action="store", type="string", dest="base0") +parser.add_option("-l", "--l0", default="-1", + help="value of segment 0 limit register", + action="store", type="string", dest="len0") +parser.add_option("-B", "--b1", default="-1", + help="value of segment 1 base register", + action="store", type="string", dest="base1") +parser.add_option("-L", "--l1", default="-1", + help="value of segment 1 limit register", + action="store", type="string", dest="len1") +parser.add_option("-c", help="compute answers for me", + action="store_true", default=False, dest="solve") + + +(options, args) = parser.parse_args() + +print "ARG seed", options.seed +print "ARG address space size", options.asize +print "ARG phys mem size", options.psize +print "" + +random.seed(options.seed) +asize = convert(options.asize) +psize = convert(options.psize) +addresses = str(options.addresses) + +if psize <= 1: + print 'Error: must specify a non-zero physical memory size.' + exit(1) + +if asize == 0: + print 'Error: must specify a non-zero address-space size.' + exit(1) + +if psize <= asize: + print 'Error: physical memory size must be GREATER than address space size (for this simulation)' + exit(1) + +# +# need to generate base, bounds for segment registers +# +len0 = convert(options.len0) +len1 = convert(options.len1) +base0 = convert(options.base0) +base1 = convert(options.base1) + +if len0 == -1: + len0 = int(asize/4.0 + (asize/4.0 * random.random())) + +if len1 == -1: + len1 = int(asize/4.0 + (asize/4.0 * random.random())) + +# now have to find room for them +if base0 == -1: + done = 0 + while done == 0: + base0 = int(psize * random.random()) + if (base0 + len0) < psize: + done = 1 + +# internally, base1 points to the lower address, and base1+len1 the higher address +# (this differs from what the user would pass in, for example) +if base1 == -1: + done = 0 + while done == 0: + base1 = int(psize * random.random()) + if (base1 + len1) < psize: + if (base1 > (base0 + len0)) or ((base1 + len1) < base0): + done = 1 +else: + base1 = base1 - len1 + + +if len0 > asize/2.0 or len1 > asize/2.0: + print 'Error: length register is too large for this address space' + exit(1) + + +print 'Segment register information:' +print '' +print ' Segment 0 base (grows positive) : 0x%08x (decimal %d)' % (base0, base0) +print ' Segment 0 limit : %d' % (len0) +print '' +print ' Segment 1 base (grows negative) : 0x%08x (decimal %d)' % (base1+len1, base1+len1) +print ' Segment 1 limit : %d' % (len1) +print '' +nbase1 = base1 + len1 + +if (len0 + base0) > (base1) and (base1 > base0): + print 'Error: segments overlap in physical memory' + exit(1) + + +addrList = [] +if addresses == '-1': + # need to generate addresses + for i in range(0, options.num): + n = int(asize * random.random()) + addrList.append(n) +else: + addrList = addresses.split(',') + +# +# now, need to generate virtual address trace +# +print 'Virtual Address Trace' +i = 0 +for vStr in addrList: + # vaddr = int(asize * random.random()) + vaddr = int(vStr) + if vaddr < 0 or vaddr >= asize: + print 'Error: virtual address %d cannot be generated in an address space of size %d' % (vaddr, asize) + exit(1) + if options.solve == False: + print ' VA %2d: 0x%08x (decimal: %4d) --> PA or segmentation violation?' % (i, vaddr, vaddr) + else: + paddr = 0 + if (vaddr >= (asize / 2)): + # seg 1 + paddr = nbase1 + (vaddr - asize) + if paddr < base1: + print ' VA %2d: 0x%08x (decimal: %4d) --> SEGMENTATION VIOLATION (SEG1)' % (i, vaddr, vaddr) + else: + print ' VA %2d: 0x%08x (decimal: %4d) --> VALID in SEG1: 0x%08x (decimal: %4d)' % (i, vaddr, vaddr, paddr, paddr) + else: + # seg 0 + if (vaddr >= len0): + print ' VA %2d: 0x%08x (decimal: %4d) --> SEGMENTATION VIOLATION (SEG0)' % (i, vaddr, vaddr) + else: + paddr = vaddr + base0 + print ' VA %2d: 0x%08x (decimal: %4d) --> VALID in SEG0: 0x%08x (decimal: %4d)' % (i, vaddr, vaddr, paddr, paddr) + i += 1 + +print '' + +if options.solve == False: + print 'For each virtual address, either write down the physical address it translates to' + print 'OR write down that it is an out-of-bounds address (a segmentation violation). For' + print 'this problem, you should assume a simple address space with two segments: the top' + print 'bit of the virtual address can thus be used to check whether the virtual address' + print 'is in segment 0 (topbit=0) or segment 1 (topbit=1). Note that the base/limit pairs' + print 'given to you grow in different directions, depending on the segment, i.e., segment 0' + print 'grows in the positive direction, whereas segment 1 in the negative. ' + print '' + + + + + diff --git a/related_info/ostep/ostep3-malloc.md b/related_info/ostep/ostep3-malloc.md new file mode 100644 index 0000000..1d2f610 --- /dev/null +++ b/related_info/ostep/ostep3-malloc.md @@ -0,0 +1,162 @@ + +This program, malloc.py, allows you to see how a simple memory allocator +works. Here are the options that you have at your disposal: + + -h, --help show this help message and exit + -s SEED, --seed=SEED the random seed + -S HEAPSIZE, --size=HEAPSIZE + size of the heap + -b BASEADDR, --baseAddr=BASEADDR + base address of heap + -H HEADERSIZE, --headerSize=HEADERSIZE + size of the header + -a ALIGNMENT, --alignment=ALIGNMENT + align allocated units to size; -1->no align + -p POLICY, --policy=POLICY + list search (BEST, WORST, FIRST) + -l ORDER, --listOrder=ORDER + list order (ADDRSORT, SIZESORT+, SIZESORT-, INSERT-FRONT, INSERT-BACK) + -C, --coalesce coalesce the free list? + -n OPSNUM, --numOps=OPSNUM + number of random ops to generate + -r OPSRANGE, --range=OPSRANGE + max alloc size + -P OPSPALLOC, --percentAlloc=OPSPALLOC + percent of ops that are allocs + -A OPSLIST, --allocList=OPSLIST + instead of random, list of ops (+10,-0,etc) + -c, --compute compute answers for me + +One way to use it is to have the program generate some random allocation/free +operations and for you to see if you can figure out what the free list would +look like, as well as the success or failure of each operation. + +Here is a simple example: + +prompt> ./malloc.py -S 100 -b 1000 -H 4 -a 4 -l ADDRSORT -p BEST -n 5 + +ptr[0] = Alloc(3) returned ? +List? + +Free(ptr[0]) returned ? +List? + +ptr[1] = Alloc(5) returned ? +List? + +Free(ptr[1]) returned ? +List? + +ptr[2] = Alloc(8) returned ? +List? + + +In this example, we specify a heap of size 100 bytes (-S 100), starting at +address 1000 (-b 1000). We specify an additional 4 bytes of header per +allocated block (-H 4), and make sure each allocated space rounds up to the +nearest 4-byte free chunk in size (-a 4). We specify that the free list be +kept ordered by address (increasing). Finally, we specify a "best fit" +free-list searching policy (-p BEST), and ask for 5 random operations to be +generated (-n 5). The results of running this are above; your job is to figure +out what each allocation/free operation returns, as well as the state of the +free list after each operation. + +Here we look at the results by using the -c option. + +prompt> ./malloc.py -S 100 -b 1000 -H 4 -a 4 -l ADDRSORT -p BEST -n 5 -c + +ptr[0] = Alloc(3) returned 1004 (searched 1 elements) +Free List [ Size 1 ]: [ addr:1008 sz:92 ] + +Free(ptr[0]) returned 0 +Free List [ Size 2 ]: [ addr:1000 sz:8 ] [ addr:1008 sz:92 ] + +ptr[1] = Alloc(5) returned 1012 (searched 2 elements) +Free List [ Size 2 ]: [ addr:1000 sz:8 ] [ addr:1020 sz:80 ] + +Free(ptr[1]) returned 0 +Free List [ Size 3 ]: [ addr:1000 sz:8 ] [ addr:1008 sz:12 ] [ addr:1020 sz:80 ] + +ptr[2] = Alloc(8) returned 1012 (searched 3 elements) +Free List [ Size 2 ]: [ addr:1000 sz:8 ] [ addr:1020 sz:80 ] + +As you can see, the first allocation operation (an allocation) returns the +following information: + +ptr[0] = Alloc(3) returned 1004 (searched 1 elements) +Free List [ Size 1 ]: [ addr:1008 sz:92 ] + +Because the initial state of the free list is just one large element, it is +easy to guess that the Alloc(3) request will succeed. Further, it will just +return the first chunk of memory and make the remainder into a free list. The +pointer returned will be just beyond the header (address:1004), and the +allocated space is rounded up to 4 bytes, leaving the free list with 92 bytes +starting at 1008. + +The next operation is a Free, of "ptr[0]" which is what stores the results of +the previous allocation request. As you can expect, this free will succeed +(thus returning "0"), and the free list now looks a little more complicated: + +Free(ptr[0]) returned 0 +Free List [ Size 2 ]: [ addr:1000 sz:8 ] [ addr:1008 sz:92 ] + +Indeed, because we are NOT coalescing the free list, we now have two elements +on it, the first being 8 bytes large and holding the just-returned space, and +the second being the 92-byte chunk. + +We can indeed turn on coalescing via the -C flag, and the result is: + +prompt> ./malloc.py -S 100 -b 1000 -H 4 -a 4 -l ADDRSORT -p BEST -n 5 -c -C +ptr[0] = Alloc(3) returned 1004 (searched 1 elements) +Free List [ Size 1 ]: [ addr:1008 sz:92 ] + +Free(ptr[0]) returned 0 +Free List [ Size 1 ]: [ addr:1000 sz:100 ] + +ptr[1] = Alloc(5) returned 1004 (searched 1 elements) +Free List [ Size 1 ]: [ addr:1012 sz:88 ] + +Free(ptr[1]) returned 0 +Free List [ Size 1 ]: [ addr:1000 sz:100 ] + +ptr[2] = Alloc(8) returned 1004 (searched 1 elements) +Free List [ Size 1 ]: [ addr:1012 sz:88 ] + +You can see that when the Free operations take place, the free list is +coalesced as expected. + +There are some other interesting options to explore: + +* -p (BEST, WORST, FIRST) + + This option lets you use these three different strategies to look for a + chunk of memory to use during an allocation request + +* -l (ADDRSORT, SIZESORT+, SIZESORT-, INSERT-FRONT, INSERT-BACK) + + This option lets you keep the free list in a particular order, + say sorted by address of the free chunk, size of free chunk (either + increasing with a + or decreasing with a -), or simply returning free + chunks to the front (INSERT-FRONT) or back (INSERT-BACK) of the free list. + +* -A (list of ops) + + This option lets you specify an exact series of requests instead + of randomly-generated ones. + + For example, running with the flag "-A +10,+10,+10,-0,-2" will allocate + three chunks of size 10 bytes (plus header), and then free the first one + ("-0") and then free the third one ("-2"). What will the free list look + like then? + +Those are the basics. Use the questions from the book chapter to explore more, +or create new and interesting questions yourself to better understand how +allocators function. + + + + + + + + diff --git a/related_info/ostep/ostep3-malloc.py b/related_info/ostep/ostep3-malloc.py new file mode 100755 index 0000000..9481ca9 --- /dev/null +++ b/related_info/ostep/ostep3-malloc.py @@ -0,0 +1,238 @@ +#! /usr/bin/env python + +import random +from optparse import OptionParser + +class malloc: + def __init__(self, size, start, headerSize, policy, order, coalesce, align): + # size of space + self.size = size + + # info about pretend headers + self.headerSize = headerSize + + # init free list + self.freelist = [] + self.freelist.append((start, size)) + + # keep track of ptr to size mappings + self.sizemap = {} + + # policy + self.policy = policy + assert(self.policy in ['FIRST', 'BEST', 'WORST']) + + # list ordering + self.returnPolicy = order + assert(self.returnPolicy in ['ADDRSORT', 'SIZESORT+', 'SIZESORT-', 'INSERT-FRONT', 'INSERT-BACK']) + + # this does a ridiculous full-list coalesce, but that is ok + self.coalesce = coalesce + + # alignment (-1 if no alignment) + self.align = align + assert(self.align == -1 or self.align > 0) + + def addToMap(self, addr, size): + assert(addr not in self.sizemap) + self.sizemap[addr] = size + # print 'adding', addr, 'to map of size', size + + def malloc(self, size): + if self.align != -1: + left = size % self.align + if left != 0: + diff = self.align - left + else: + diff = 0 + # print 'aligning: adding %d to %d' % (diff, size) + size += diff + + size += self.headerSize + + bestIdx = -1 + if self.policy == 'BEST': + bestSize = self.size + 1 + elif self.policy == 'WORST' or self.policy == 'FIRST': + bestSize = -1 + + count = 0 + + for i in range(len(self.freelist)): + eaddr, esize = self.freelist[i][0], self.freelist[i][1] + count += 1 + if esize >= size and ((self.policy == 'BEST' and esize < bestSize) or + (self.policy == 'WORST' and esize > bestSize) or + (self.policy == 'FIRST')): + bestAddr = eaddr + bestSize = esize + bestIdx = i + if self.policy == 'FIRST': + break + + if bestIdx != -1: + if bestSize > size: + # print 'SPLIT', bestAddr, size + self.freelist[bestIdx] = (bestAddr + size, bestSize - size) + self.addToMap(bestAddr, size) + elif bestSize == size: + # print 'PERFECT MATCH (no split)', bestAddr, size + self.freelist.pop(bestIdx) + self.addToMap(bestAddr, size) + else: + abort('should never get here') + return (bestAddr, count) + + # print '*** FAILED TO FIND A SPOT', size + return (-1, count) + + def free(self, addr): + # simple back on end of list, no coalesce + if addr not in self.sizemap: + return -1 + + size = self.sizemap[addr] + if self.returnPolicy == 'INSERT-BACK': + self.freelist.append((addr, size)) + elif self.returnPolicy == 'INSERT-FRONT': + self.freelist.insert(0, (addr, size)) + elif self.returnPolicy == 'ADDRSORT': + self.freelist.append((addr, size)) + self.freelist = sorted(self.freelist, key=lambda e: e[0]) + elif self.returnPolicy == 'SIZESORT+': + self.freelist.append((addr, size)) + self.freelist = sorted(self.freelist, key=lambda e: e[1], reverse=False) + elif self.returnPolicy == 'SIZESORT-': + self.freelist.append((addr, size)) + self.freelist = sorted(self.freelist, key=lambda e: e[1], reverse=True) + + # not meant to be an efficient or realistic coalescing... + if self.coalesce == True: + self.newlist = [] + self.curr = self.freelist[0] + for i in range(1, len(self.freelist)): + eaddr, esize = self.freelist[i] + if eaddr == (self.curr[0] + self.curr[1]): + self.curr = (self.curr[0], self.curr[1] + esize) + else: + self.newlist.append(self.curr) + self.curr = eaddr, esize + self.newlist.append(self.curr) + self.freelist = self.newlist + + del self.sizemap[addr] + return 0 + + def dump(self): + print 'Free List [ Size %d ]: ' % len(self.freelist), + for e in self.freelist: + print '[ addr:%d sz:%d ]' % (e[0], e[1]), + print '' + + +# +# main program +# +parser = OptionParser() + +parser.add_option('-s', '--seed', default=0, help='the random seed', action='store', type='int', dest='seed') +parser.add_option('-S', '--size', default=100, help='size of the heap', action='store', type='int', dest='heapSize') +parser.add_option('-b', '--baseAddr', default=1000, help='base address of heap', action='store', type='int', dest='baseAddr') +parser.add_option('-H', '--headerSize', default=0, help='size of the header', action='store', type='int', dest='headerSize') +parser.add_option('-a', '--alignment', default=-1, help='align allocated units to size; -1->no align', action='store', type='int', dest='alignment') +parser.add_option('-p', '--policy', default='BEST', help='list search (BEST, WORST, FIRST)', action='store', type='string', dest='policy') +parser.add_option('-l', '--listOrder', default='ADDRSORT', help='list order (ADDRSORT, SIZESORT+, SIZESORT-, INSERT-FRONT, INSERT-BACK)', action='store', type='string', dest='order') +parser.add_option('-C', '--coalesce', default=False, help='coalesce the free list?', action='store_true', dest='coalesce') +parser.add_option('-n', '--numOps', default=10, help='number of random ops to generate', action='store', type='int', dest='opsNum') +parser.add_option('-r', '--range', default=10, help='max alloc size', action='store', type='int', dest='opsRange') +parser.add_option('-P', '--percentAlloc',default=50, help='percent of ops that are allocs', action='store', type='int', dest='opsPAlloc') +parser.add_option('-A', '--allocList', default='', help='instead of random, list of ops (+10,-0,etc)', action='store', type='string', dest='opsList') +parser.add_option('-c', '--compute', default=False, help='compute answers for me', action='store_true', dest='solve') + +(options, args) = parser.parse_args() + +m = malloc(int(options.heapSize), int(options.baseAddr), int(options.headerSize), + options.policy, options.order, options.coalesce, options.alignment) + +percent = int(options.opsPAlloc) / 100.0 + +random.seed(int(options.seed)) +p = {} +L = [] +assert(percent > 0) + +if options.opsList == '': + c = 0 + j = 0 + while j < int(options.opsNum): + pr = False + if random.random() < percent: + size = int(random.random() * int(options.opsRange)) + 1 + ptr, cnt = m.malloc(size) + if ptr != -1: + p[c] = ptr + L.append(c) + print 'ptr[%d] = Alloc(%d)' % (c, size), + if options.solve == True: + print ' returned %d (searched %d elements)' % (ptr + options.headerSize, cnt) + else: + print ' returned ?' + c += 1 + j += 1 + pr = True + else: + if len(p) > 0: + # pick random one to delete + d = int(random.random() * len(L)) + rc = m.free(p[L[d]]) + print 'Free(ptr[%d])' % L[d], + if options.solve == True: + print 'returned %d' % rc + else: + print 'returned ?' + del p[L[d]] + del L[d] + # print 'DEBUG p', p + # print 'DEBUG L', L + pr = True + j += 1 + if pr: + if options.solve == True: + m.dump() + else: + print 'List? ' + print '' +else: + c = 0 + for op in options.opsList.split(','): + if op[0] == '+': + # allocation! + size = int(op.split('+')[1]) + ptr, cnt = m.malloc(size) + if ptr != -1: + p[c] = ptr + print 'ptr[%d] = Alloc(%d)' % (c, size), + if options.solve == True: + print ' returned %d (searched %d elements)' % (ptr, cnt) + else: + print ' returned ?' + c += 1 + elif op[0] == '-': + # free + index = int(op.split('-')[1]) + if index >= len(p): + print 'Invalid Free: Skipping' + continue + print 'Free(ptr[%d])' % index, + rc = m.free(p[index]) + if options.solve == True: + print 'returned %d' % rc + else: + print 'returned ?' + else: + abort('badly specified operand: must be +Size or -Index') + if options.solve == True: + m.dump() + else: + print 'List?' + print '' diff --git a/related_info/ostep/ostep4-paging-linear-translate.md b/related_info/ostep/ostep4-paging-linear-translate.md new file mode 100644 index 0000000..334a626 --- /dev/null +++ b/related_info/ostep/ostep4-paging-linear-translate.md @@ -0,0 +1,131 @@ + +In this homework, you will use a simple program, which is known as +paging-linear-translate.py, to see if you understand how simple +virtual-to-physical address translation works with linear page tables. To run +the program, remember to either type just the name of the program +(./paging-linear-translate.py) or possibly this (python +paging-linear-translate.py). When you run it with the -h (help) flag, you +see: + +Usage: paging-linear-translate.py [options] + +Options: +-h, --help show this help message and exit +-s SEED, --seed=SEED the random seed +-a ASIZE, --asize=ASIZE + address space size (e.g., 16, 64k, ...) +-p PSIZE, --physmem=PSIZE + physical memory size (e.g., 16, 64k, ...) +-P PAGESIZE, --pagesize=PAGESIZE + page size (e.g., 4k, 8k, ...) +-n NUM, --addresses=NUM number of virtual addresses to generate +-u USED, --used=USED percent of address space that is used +-v verbose mode +-c compute answers for me + +First, run the program without any arguments: + +ARG seed 0 +ARG address space size 16k +ARG phys mem size 64k +ARG page size 4k +ARG verbose False + +The format of the page table is simple: +The high-order (left-most) bit is the VALID bit. + If the bit is 1, the rest of the entry is the PFN. + If the bit is 0, the page is not valid. +Use verbose mode (-v) if you want to print the VPN # by +each entry of the page table. + +Page Table (from entry 0 down to the max size) + 0x8000000c + 0x00000000 + 0x00000000 + 0x80000006 + +Virtual Address Trace + VA 0: 0x00003229 (decimal: 12841) --> PA or invalid? + VA 1: 0x00001369 (decimal: 4969) --> PA or invalid? + VA 2: 0x00001e80 (decimal: 7808) --> PA or invalid? + VA 3: 0x00002556 (decimal: 9558) --> PA or invalid? + VA 4: 0x00003a1e (decimal: 14878) --> PA or invalid? + +For each virtual address, write down the physical address it +translates to OR write down that it is an out-of-bounds +address (e.g., a segmentation fault). + +As you can see, what the program provides for you is a page table for a +particular process (remember, in a real system with linear page tables, there +is one page table per process; here we just focus on one process, its address +space, and thus a single page table). The page table tells you, for each +virtual page number (VPN) of the address space, that the virtual page is +mapped to a particular physical frame number (PFN) and thus valid, or not +valid. + +The format of the page-table entry is simple: the left-most (high-order) bit +is the valid bit; the remaining bits, if valid is 1, is the PFN. + +In the example above, the page table maps VPN 0 to PFN 0xc (decimal 12), VPN 3 +to PFN 0x6 (decimal 6), and leaves the other two virtual pages, 1 and 2, as +not valid. + +Because the page table is a linear array, what is printed above is a replica +of what you would see in memory if you looked at the bits yourself. However, +it is sometimes easier to use this simulator if you run with the verbose flag +(-v); this flag also prints out the VPN (index) into the page table. From the +example above, run with the -v flag: + +Page Table (from entry 0 down to the max size) + [ 0] 0x8000000c + [ 1] 0x00000000 + [ 2] 0x00000000 + [ 3] 0x80000006 + +Your job, then, is to use this page table to translate the virtual addresses +given to you in the trace to physical addresses. Let's look at the first one: +VA 0x3229. To translate this virtual address into a physical address, we first +have to break it up into its constituent components: a virtual page number and +an offset. We do this by noting down the size of the address space and the +page size. In this example, the address space is set to 16KB (a very small +address space) and the page size is 4KB. Thus, we know that there are 14 bits +in the virtual address, and that the offset is 12 bits, leaving 2 bits for the +VPN. Thus, with our address 0x3229, which is binary 11 0010 0010 1001, we know +the top two bits specify the VPN. Thus, 0x3229 is on virtual page 3 with an +offset of 0x229. + +We next look in the page table to see if VPN 3 is valid and mapped to some +physical frame or invalid, and we see that it is indeed valid (the high bit is +1) and mapped to physical page 6. Thus, we can form our final physical address +by taking the physical page 6 and adding it onto the offset, as follows: +0x6000 (the physical page, shifted into the proper spot) OR 0x0229 (the +offset), yielding the final physical address: 0x6229. Thus, we can see that +virtual address 0x3229 translates to physical address 0x6229 in this example. + +To see the rest of the solutions (after you have computed them yourself!), +just run with the -c flag (as always): + +... +VA 0: 00003229 (decimal: 12841) --> 00006229 (25129) [VPN 3] +VA 1: 00001369 (decimal: 4969) --> Invalid (VPN 1 not valid) +VA 2: 00001e80 (decimal: 7808) --> Invalid (VPN 1 not valid) +VA 3: 00002556 (decimal: 9558) --> Invalid (VPN 2 not valid) +VA 4: 00003a1e (decimal: 14878) --> 00006a1e (27166) [VPN 3] + +Of course, you can change many of these parameters to make more interesting +problems. Run the program with the -h flag to see what options there are: + +- The -s flag changes the random seed and thus generates different + page table values as well as different virtual addresses to translate. +- The -a flag changes the size of the address space. +- The -p flag changes the size of physical memory. +- The -P flag changes the size of a page. +- The -n flag can be used to generate more addresses to translate + (instead of the default 5). +- The -u flag changes the fraction of mappings that are valid, from + 0% (-u 0) up to 100% (-u 100). The default is 50, which means + that roughly 1/2 of the pages in the virtual address space will be valid. +- The -v flag prints out the VPN numbers to make your life easier. + + + diff --git a/related_info/ostep/ostep4-paging-linear-translate.py b/related_info/ostep/ostep4-paging-linear-translate.py new file mode 100755 index 0000000..5bdb4eb --- /dev/null +++ b/related_info/ostep/ostep4-paging-linear-translate.py @@ -0,0 +1,192 @@ +#! /usr/bin/env python + +import sys +from optparse import OptionParser +import random +import math + +def mustbepowerof2(bits, size, msg): + if math.pow(2,bits) != size: + print 'Error in argument: %s' % msg + sys.exit(1) + +def mustbemultipleof(bignum, num, msg): + if (int(float(bignum)/float(num)) != (int(bignum) / int(num))): + print 'Error in argument: %s' % msg + sys.exit(1) + +def convert(size): + length = len(size) + lastchar = size[length-1] + if (lastchar == 'k') or (lastchar == 'K'): + m = 1024 + nsize = int(size[0:length-1]) * m + elif (lastchar == 'm') or (lastchar == 'M'): + m = 1024*1024 + nsize = int(size[0:length-1]) * m + elif (lastchar == 'g') or (lastchar == 'G'): + m = 1024*1024*1024 + nsize = int(size[0:length-1]) * m + else: + nsize = int(size) + return nsize + + +# +# main program +# +parser = OptionParser() +parser.add_option('-A', '--addresses', default='-1', + help='a set of comma-separated pages to access; -1 means randomly generate', + action='store', type='string', dest='addresses') +parser.add_option('-s', '--seed', default=0, help='the random seed', action='store', type='int', dest='seed') +parser.add_option('-a', '--asize', default='16k', help='address space size (e.g., 16, 64k, 32m, 1g)', action='store', type='string', dest='asize') +parser.add_option('-p', '--physmem', default='64k', help='physical memory size (e.g., 16, 64k, 32m, 1g)', action='store', type='string', dest='psize') +parser.add_option('-P', '--pagesize', default='4k', help='page size (e.g., 4k, 8k, whatever)', action='store', type='string', dest='pagesize') +parser.add_option('-n', '--numaddrs', default=5, help='number of virtual addresses to generate', action='store', type='int', dest='num') +parser.add_option('-u', '--used', default=50, help='percent of virtual address space that is used', action='store', type='int', dest='used') +parser.add_option('-v', help='verbose mode', action='store_true', default=False, dest='verbose') +parser.add_option('-c', help='compute answers for me', action='store_true', default=False, dest='solve') + + +(options, args) = parser.parse_args() + +print 'ARG seed', options.seed +print 'ARG address space size', options.asize +print 'ARG phys mem size', options.psize +print 'ARG page size', options.pagesize +print 'ARG verbose', options.verbose +print 'ARG addresses', options.addresses +print '' + +random.seed(options.seed) + +asize = convert(options.asize) +psize = convert(options.psize) +pagesize = convert(options.pagesize) +addresses = str(options.addresses) + +if psize <= 1: + print 'Error: must specify a non-zero physical memory size.' + exit(1) + +if asize < 1: + print 'Error: must specify a non-zero address-space size.' + exit(1) + +if psize <= asize: + print 'Error: physical memory size must be GREATER than address space size (for this simulation)' + exit(1) + +if psize >= convert('1g') or asize >= convert('1g'): + print 'Error: must use smaller sizes (less than 1 GB) for this simulation.' + exit(1) + +mustbemultipleof(asize, pagesize, 'address space must be a multiple of the pagesize') +mustbemultipleof(psize, pagesize, 'physical memory must be a multiple of the pagesize') + +# print some useful info, like the darn page table +pages = psize / pagesize; +import array +used = array.array('i') +pt = array.array('i') +for i in range(0,pages): + used.insert(i,0) +vpages = asize / pagesize + +# now, assign some pages of the VA +vabits = int(math.log(float(asize))/math.log(2.0)) +mustbepowerof2(vabits, asize, 'address space must be a power of 2') +pagebits = int(math.log(float(pagesize))/math.log(2.0)) +mustbepowerof2(pagebits, pagesize, 'page size must be a power of 2') +vpnbits = vabits - pagebits +pagemask = (1 << pagebits) - 1 + +# import ctypes +# vpnmask = ctypes.c_uint32(~pagemask).value +vpnmask = 0xFFFFFFFF & ~pagemask +#if vpnmask2 != vpnmask: +# print 'ERROR' +# exit(1) +# print 'va:%d page:%d vpn:%d -- %08x %08x' % (vabits, pagebits, vpnbits, vpnmask, pagemask) + +print '' +print 'The format of the page table is simple:' +print 'The high-order (left-most) bit is the VALID bit.' +print ' If the bit is 1, the rest of the entry is the PFN.' +print ' If the bit is 0, the page is not valid.' +print 'Use verbose mode (-v) if you want to print the VPN # by' +print 'each entry of the page table.' +print '' + +print 'Page Table (from entry 0 down to the max size)' +for v in range(0,vpages): + done = 0 + while done == 0: + if ((random.random() * 100.0) > (100.0 - float(options.used))): + u = int(pages * random.random()) + if used[u] == 0: + done = 1 + # print '%8d - %d' % (v, u) + if options.verbose == True: + print ' [%8d] ' % v, + else: + print ' ', + print '0x%08x' % (0x80000000 | u) + pt.insert(v,u) + else: + # print '%8d - not valid' % v + if options.verbose == True: + print ' [%8d] ' % v, + else: + print ' ', + print '0x%08x' % 0 + pt.insert(v,-1) + done = 1 +print '' + + +# +# now, need to generate virtual address trace +# + +addrList = [] +if addresses == '-1': + # need to generate addresses + for i in range(0, options.num): + n = int(asize * random.random()) + addrList.append(n) +else: + addrList = addresses.split(',') + + +print 'Virtual Address Trace' +for vStr in addrList: + # vaddr = int(asize * random.random()) + vaddr = int(vStr) + if options.solve == False: + print ' VA 0x%08x (decimal: %8d) --> PA or invalid address?' % (vaddr, vaddr) + else: + paddr = 0 + # split vaddr into VPN | offset + vpn = (vaddr & vpnmask) >> pagebits + if pt[vpn] < 0: + print ' VA 0x%08x (decimal: %8d) --> Invalid (VPN %d not valid)' % (vaddr, vaddr, vpn) + else: + pfn = pt[vpn] + offset = vaddr & pagemask + paddr = (pfn << pagebits) | offset + print ' VA 0x%08x (decimal: %8d) --> %08x (decimal %8d) [VPN %d]' % (vaddr, vaddr, paddr, paddr, vpn) +print '' + +if options.solve == False: + print 'For each virtual address, write down the physical address it translates to' + print 'OR write down that it is an out-of-bounds address (e.g., segfault).' + print '' + + + + + + + diff --git a/related_info/ostep/ostep5-paging-multilevel-translate.md b/related_info/ostep/ostep5-paging-multilevel-translate.md new file mode 100644 index 0000000..c0836e7 --- /dev/null +++ b/related_info/ostep/ostep5-paging-multilevel-translate.md @@ -0,0 +1,71 @@ + +This fun little homework tests if you understand how a multi-level page table +works. And yes, there is some debate over the use of the term fun in the +previous sentence. The program is called: + paging-multilevel-translate.py + +Some basic assumptions: + +- The page size is an unrealistically-small 32 bytes +- The virtual address space for the process in question (assume there is + only one) is 1024 pages, or 32 KB +- physical memory consists of 128 pages + +Thus, a virtual address needs 15 bits (5 for the offset, 10 for the VPN). +A physical address requires 12 bits (5 offset, 7 for the PFN). + +The system assumes a multi-level page table. Thus, the upper five bits of a virtual +address are used to index into a page directory; the page directory entry (PDE), if valid, +points to a page of the page table. Each page table page holds 32 page-table entries +(PTEs). Each PTE, if valid, holds the desired translation (physical frame number, or PFN) +of the virtual page in question. + +The format of a PTE is thus: + + VALID | PFN6 ... PFN0 + +and is thus 8 bits or 1 byte. + +The format of a PDE is essentially identical: + + VALID | PT6 ... PT0 + +You are given two pieces of information to begin with. + +First, you are given the value of the page directory base register (PDBR), +which tells you which page the page directory is located upon. + +Second, you are given a complete dump of each page of memory. A page dump +looks like this: + + page 0: 08 00 01 15 11 1d 1d 1c 01 17 15 14 16 1b 13 0b ... + page 1: 19 05 1e 13 02 16 1e 0c 15 09 06 16 00 19 10 03 ... + page 2: 1d 07 11 1b 12 05 07 1e 09 1a 18 17 16 18 1a 01 ... + ... + +which shows the 32 bytes found on pages 0, 1, 2, and so forth. The first byte +(0th byte) on page 0 has the value 0x08, the second is 0x00, the third 0x01, +and so forth. + +You are then given a list of virtual addresses to translate. + +Use the PDBR to find the relevant page table entries for this virtual page. +Then find if it is valid. If so, use the translation to form a final physical +address. Using this address, you can find the VALUE that the memory reference +is looking for. + +Of course, the virtual address may not be valid and thus generate a fault. + +Some useful options: + + -s SEED, --seed=SEED the random seed + -n NUM, --addresses=NUM number of virtual addresses to generate + -c, --solve compute answers for me + +Change the seed to get different problems, as always. + +Change the number of virtual addresses generated to do more translations +for a given memory dump. + +Use -c (or --solve) to show the solutions. + diff --git a/related_info/ostep/ostep5-paging-multilevel-translate.py b/related_info/ostep/ostep5-paging-multilevel-translate.py new file mode 100755 index 0000000..60d65ef --- /dev/null +++ b/related_info/ostep/ostep5-paging-multilevel-translate.py @@ -0,0 +1,263 @@ +#! /usr/bin/env python + +import sys +from optparse import OptionParser +import random +import math + +def convert(size): + length = len(size) + lastchar = size[length-1] + if (lastchar == 'k') or (lastchar == 'K'): + m = 1024 + nsize = int(size[0:length-1]) * m + elif (lastchar == 'm') or (lastchar == 'M'): + m = 1024*1024 + nsize = int(size[0:length-1]) * m + elif (lastchar == 'g') or (lastchar == 'G'): + m = 1024*1024*1024 + nsize = int(size[0:length-1]) * m + else: + nsize = int(size) + return nsize + +def roundup(size): + value = 1.0 + while value < size: + value = value * 2.0 + return value + + +class OS: + def __init__(self): + # 4k phys memory (128 pages) + self.pageSize = 32 + self.physPages = 128 + self.physMem = self.pageSize * self.physPages + self.vaPages = 1024 + self.vaSize = self.pageSize * self.vaPages + self.pteSize = 1 + self.pageBits = 5 # log of page size + + # os tracks + self.usedPages = [] + self.usedPagesCount = 0 + self.maxPageCount = self.physMem / self.pageSize + + # no pages used (yet) + for i in range(0, self.maxPageCount): + self.usedPages.append(0) + + # set contents of memory to 0, too + self.memory = [] + for i in range(0, self.physMem): + self.memory.append(0) + + # associative array of pdbr's (indexed by PID) + self.pdbr = {} + + # mask is 11111 00000 00000 --> 0111 1100 0000 0000 + self.PDE_MASK = 0x7c00 + self.PDE_SHIFT = 10 + + # 00000 11111 00000 -> 000 0011 1110 0000 + self.PTE_MASK = 0x03e0 + self.PTE_SHIFT = 5 + + self.VPN_MASK = self.PDE_MASK | self.PTE_MASK + self.VPN_SHIFT = self.PTE_SHIFT + + # grabs the last five bits of a virtual address + self.OFFSET_MASK = 0x1f + + def findFree(self): + assert(self.usedPagesCount < self.maxPageCount) + look = int(random.random() * self.maxPageCount) + while self.usedPages[look] == 1: + look = int(random.random() * self.maxPageCount) + self.usedPagesCount = self.usedPagesCount + 1 + self.usedPages[look] = 1 + return look + + def initPageDir(self, whichPage): + whichByte = whichPage << self.pageBits + for i in range(whichByte, whichByte + self.pageSize): + self.memory[i] = 0x7f + + def initPageTablePage(self, whichPage): + self.initPageDir(whichPage) + + def getPageTableEntry(self, virtualAddr, ptePage, printStuff): + pteBits = (virtualAddr & self.PTE_MASK) >> self.PTE_SHIFT + pteAddr = (ptePage << self.pageBits) | pteBits + pte = self.memory[pteAddr] + valid = (pte & 0x80) >> 7 + pfn = (pte & 0x7f) + if printStuff == True: + print ' --> pte index:0x%x pte contents:(valid %d, pfn 0x%02x)' % (pteBits, valid, pfn) + return (valid, pfn, pteAddr) + + def getPageDirEntry(self, pid, virtualAddr, printStuff): + pageDir = self.pdbr[pid] + pdeBits = (virtualAddr & self.PDE_MASK) >> self.PDE_SHIFT + pdeAddr = (pageDir << self.pageBits) | pdeBits + pde = self.memory[pdeAddr] + valid = (pde & 0x80) >> 7 + ptPtr = (pde & 0x7f) + if printStuff == True: + print ' --> pde index:0x%x pde contents:(valid %d, pfn 0x%02x)' % (pdeBits, valid, ptPtr) + return (valid, ptPtr, pdeAddr) + + def setPageTableEntry(self, pteAddr, physicalPage): + self.memory[pteAddr] = 0x80 | physicalPage + + def setPageDirEntry(self, pdeAddr, physicalPage): + self.memory[pdeAddr] = 0x80 | physicalPage + + def allocVirtualPage(self, pid, virtualPage, physicalPage): + # make it into a virtual address, as everything uses this (and not VPN) + virtualAddr = virtualPage << self.pageBits + (valid, ptPtr, pdeAddr) = self.getPageDirEntry(pid, virtualAddr, False) + if valid == 0: + # must allocate a page of the page table now, and have the PD point to it + assert(ptPtr == 127) + ptePage = self.findFree() + self.setPageDirEntry(pdeAddr, ptePage) + self.initPageTablePage(ptePage) + else: + # otherwise, just extract page number of page table page + ptePage = ptPtr + # now, look up page table entry too, and mark it valid and fill in translation + (valid, pfn, pteAddr) = self.getPageTableEntry(virtualAddr, ptePage, False) + assert(valid == 0) + assert(pfn == 127) + self.setPageTableEntry(pteAddr, physicalPage) + + # -2 -> PTE fault, -1 means PDE fault + def translate(self, pid, virtualAddr): + (valid, ptPtr, pdeAddr) = self.getPageDirEntry(pid, virtualAddr, True) + if valid == 1: + ptePage = ptPtr + (valid, pfn, pteAddr) = self.getPageTableEntry(virtualAddr, ptePage, True) + if valid == 1: + offset = (virtualAddr & self.OFFSET_MASK) + paddr = (pfn << self.pageBits) | offset + # print ' --> pfn: %02x offset: %x' % (pfn, offset) + return paddr + else: + return -2 + return -1 + + def fillPage(self, whichPage): + for j in range(0, self.pageSize): + self.memory[(whichPage * self.pageSize) + j] = int(random.random() * 31) + + def procAlloc(self, pid, numPages): + # need a PDBR: find one somewhere in memory + pageDir = self.findFree() + # print '**ALLOCATE** page dir', pageDir + self.pdbr[pid] = pageDir + self.initPageDir(pageDir) + + used = {} + for vp in range(0, self.vaPages): + used[vp] = 0 + allocatedVPs = [] + + for vp in range(0, numPages): + vp = int(random.random() * self.vaPages) + while used[vp] == 1: + vp = int(random.random() * self.vaPages) + assert(used[vp] == 0) + used[vp] = 1 + allocatedVPs.append(vp) + pp = self.findFree() + # print '**ALLOCATE** page', pp + # print ' trying to map vp:%08x to pp:%08x' % (vp, pp) + self.allocVirtualPage(pid, vp, pp) + self.fillPage(pp) + return allocatedVPs + + def dumpPage(self, whichPage): + i = whichPage + for j in range(0, self.pageSize): + print self.memory[(i * self.pageSize) + j], + print '' + + def memoryDump(self): + for i in range(0, self.physMem / self.pageSize): + print 'page %3d:' % i, + for j in range(0, self.pageSize): + print '%02x' % self.memory[(i * self.pageSize) + j], + print '' + + def getPDBR(self, pid): + return self.pdbr[pid] + + def getValue(self, addr): + return self.memory[addr] + +# allocate some processes in memory +# allocate some multi-level page tables in memory +# make a bit of a mystery: +# can examine PDBR (PFN of current proc's page directory) +# can examine contents of any page +# fill pages with values too +# ask: when given +# LOAD VA, R1 +# what will final value will be loaded into R1? + +# +# main program +# +parser = OptionParser() +parser.add_option('-s', '--seed', default=0, help='the random seed', action='store', type='int', dest='seed') +parser.add_option('-a', '--allocated', default=64, help='number of virtual pages allocated', + action='store', type='int', dest='allocated') +parser.add_option('-n', '--addresses', default=10, help='number of virtual addresses to generate', + action='store', type='int', dest='num') +parser.add_option('-c', '--solve', help='compute answers for me', action='store_true', default=False, dest='solve') + + +(options, args) = parser.parse_args() + +print 'ARG seed', options.seed +print 'ARG allocated', options.allocated +print 'ARG num', options.num +print "" + +random.seed(options.seed) + +# do the work now +os = OS() +used = os.procAlloc(1, options.allocated) + +os.memoryDump() + +print '\nPDBR:', os.getPDBR(1), ' (decimal) [This means the page directory is held in this page]\n' + +for i in range(0, options.num): + if (random.random() * 100) > 50.0 or i >= len(used): + vaddr = int(random.random() * 1024 * 32) + else: + vaddr = (used[i] << 5) | int(random.random() * 32) + if options.solve == True: + print 'Virtual Address %04x:' % vaddr + r = os.translate(1, vaddr) + if r > -1: + print ' --> Translates to Physical Address 0x%03x --> Value: %02x' % (r, os.getValue(r)) + elif r == -1: + print ' --> Fault (page directory entry not valid)' + else: + print ' --> Fault (page table entry not valid)' + else: + print 'Virtual Address %04x: Translates To What Physical Address (And Fetches what Value)? Or Fault?' % vaddr + +print '' + +exit(0) + + + + + diff --git a/related_info/ostep/ostep6-paging-policy.md b/related_info/ostep/ostep6-paging-policy.md new file mode 100644 index 0000000..8840344 --- /dev/null +++ b/related_info/ostep/ostep6-paging-policy.md @@ -0,0 +1,119 @@ + +This simulator, paging-policy.py, allows you to play around with different +page-replacement policies. For example, let's examine how LRU performs with a +series of page references with a cache of size 3: + + 0 1 2 0 1 3 0 3 1 2 1 + +To do so, run the simulator as follows: + +prompt> ./paging-policy.py --addresses=0,1,2,0,1,3,0,3,1,2,1 + --policy=LRU --cachesize=3 -c + +And what you would see is: + +ARG addresses 0,1,2,0,1,3,0,3,1,2,1 +ARG numaddrs 10 +ARG policy LRU +ARG cachesize 3 +ARG maxpage 10 +ARG seed 0 + +Solving... + +Access: 0 MISS LRU-> [br 0]<-MRU Replace:- [br Hits:0 Misses:1] +Access: 1 MISS LRU-> [br 0, 1]<-MRU Replace:- [br Hits:0 Misses:2] +Access: 2 MISS LRU->[br 0, 1, 2]<-MRU Replace:- [br Hits:0 Misses:3] +Access: 0 HIT LRU->[br 1, 2, 0]<-MRU Replace:- [br Hits:1 Misses:3] +Access: 1 HIT LRU->[br 2, 0, 1]<-MRU Replace:- [br Hits:2 Misses:3] +Access: 3 MISS LRU->[br 0, 1, 3]<-MRU Replace:2 [br Hits:2 Misses:4] +Access: 0 HIT LRU->[br 1, 3, 0]<-MRU Replace:2 [br Hits:3 Misses:4] +Access: 3 HIT LRU->[br 1, 0, 3]<-MRU Replace:2 [br Hits:4 Misses:4] +Access: 1 HIT LRU->[br 0, 3, 1]<-MRU Replace:2 [br Hits:5 Misses:4] +Access: 2 MISS LRU->[br 3, 1, 2]<-MRU Replace:0 [br Hits:5 Misses:5] +Access: 1 HIT LRU->[br 3, 2, 1]<-MRU Replace:0 [br Hits:6 Misses:5] +] + +The complete set of possible arguments for paging-policy is listed on the +following page, and includes a number of options for varying the policy, how +addresses are specified/generated, and other important parameters such as the +size of the cache. + +prompt> ./paging-policy.py --help +Usage: paging-policy.py [options] + +Options: +-h, --help show this help message and exit +-a ADDRESSES, --addresses=ADDRESSES + a set of comma-separated pages to access; + -1 means randomly generate +-f ADDRESSFILE, --addressfile=ADDRESSFILE + a file with a bunch of addresses in it +-n NUMADDRS, --numaddrs=NUMADDRS + if -a (--addresses) is -1, this is the + number of addrs to generate +-p POLICY, --policy=POLICY + replacement policy: FIFO, LRU, LFU, OPT, + UNOPT, RAND, CLOCK +-b CLOCKBITS, --clockbits=CLOCKBITS + for CLOCK policy, how many clock bits to use +-C CACHESIZE, --cachesize=CACHESIZE + size of the page cache, in pages +-m MAXPAGE, --maxpage=MAXPAGE + if randomly generating page accesses, + this is the max page number +-s SEED, --seed=SEED random number seed +-N, --notrace do not print out a detailed trace +-c, --compute compute answers for me +] + +As usual, "-c" is used to solve a particular problem, whereas without it, the +accesses are just listed (and the program does not tell you whether or not a +particular access is a hit or miss). + +To generate a random problem, instead of using "-a/--addresses" to pass in +some page references, you can instead pass in "-n/--numaddrs" as the number of +addresses the program should randomly generate, with "-s/--seed" used to +specify a different random seed. For example: + +prompt> ./paging-policy.py -s 10 -n 3 +.. . +Assuming a replacement policy of FIFO, and a cache of size 3 pages, +figure out whether each of the following page references hit or miss +in the page cache. + +Access: 5 Hit/Miss? State of Memory? +Access: 4 Hit/Miss? State of Memory? +Access: 5 Hit/Miss? State of Memory? +] + +As you can see, in this example, we specify "-n 3" which means the program +should generate 3 random page references, which it does: 5, 7, and 5. The +random seed is also specified (10), which is what gets us those particular +numbers. After working this out yourself, have the program solve the problem +for you by passing in the same arguments but with "-c" (showing just the +relevant part here): + +prompt> ./paging-policy.py -s 10 -n 3 -c +... +Solving... + +Access: 5 MISS FirstIn-> [br 5] <-Lastin Replace:- [br Hits:0 Misses:1] +Access: 4 MISS FirstIn->[br 5, 4] <-Lastin Replace:- [br Hits:0 Misses:2] +Access: 5 HIT FirstIn->[br 5, 4] <-Lastin Replace:- [br Hits:1 Misses:2] +] + +The default policy is FIFO, though others are available, including LRU, MRU, +OPT (the optimal replacement policy, which peeks into the future to see what +is best to replace), UNOPT (which is the pessimal replacement), RAND (which +does random replacement), and CLOCK (which does the clock algorithm). The +CLOCK algorithm also takes another argument (-b), which states how many bits +should be kept per page; the more clock bits there are, the better the +algorithm should be at determining which pages to keep in memory. + +Other options include: "-C/--cachesize" which changes the size of the page +cache; "-m/--maxpage" which is the largest page number that will be used if +the simulator is generating references for you; and "-f/--addressfile" which +lets you specify a file with addresses in them, in case you wish to get traces +from a real application or otherwise use a long trace as input. + diff --git a/related_info/ostep/ostep6-paging-policy.py b/related_info/ostep/ostep6-paging-policy.py new file mode 100755 index 0000000..58ef86a --- /dev/null +++ b/related_info/ostep/ostep6-paging-policy.py @@ -0,0 +1,275 @@ +#! /usr/bin/env python + +import sys +from optparse import OptionParser +import random +import math + +def convert(size): + length = len(size) + lastchar = size[length-1] + if (lastchar == 'k') or (lastchar == 'K'): + m = 1024 + nsize = int(size[0:length-1]) * m + elif (lastchar == 'm') or (lastchar == 'M'): + m = 1024*1024 + nsize = int(size[0:length-1]) * m + elif (lastchar == 'g') or (lastchar == 'G'): + m = 1024*1024*1024 + nsize = int(size[0:length-1]) * m + else: + nsize = int(size) + return nsize + +def hfunc(index): + if index == -1: + return 'MISS' + else: + return 'HIT ' + +def vfunc(victim): + if victim == -1: + return '-' + else: + return str(victim) + +# +# main program +# +parser = OptionParser() +parser.add_option('-a', '--addresses', default='-1', help='a set of comma-separated pages to access; -1 means randomly generate', action='store', type='string', dest='addresses') +parser.add_option('-f', '--addressfile', default='', help='a file with a bunch of addresses in it', action='store', type='string', dest='addressfile') +parser.add_option('-n', '--numaddrs', default='10', help='if -a (--addresses) is -1, this is the number of addrs to generate', action='store', type='string', dest='numaddrs') +parser.add_option('-p', '--policy', default='FIFO', help='replacement policy: FIFO, LRU, OPT, UNOPT, RAND, CLOCK', action='store', type='string', dest='policy') +parser.add_option('-b', '--clockbits', default=2, help='for CLOCK policy, how many clock bits to use', action='store', type='int', dest='clockbits') +parser.add_option('-C', '--cachesize', default='3', help='size of the page cache, in pages', action='store', type='string', dest='cachesize') +parser.add_option('-m', '--maxpage', default='10', help='if randomly generating page accesses, this is the max page number', action='store', type='string', dest='maxpage') +parser.add_option('-s', '--seed', default='0', help='random number seed', action='store', type='string', dest='seed') +parser.add_option('-N', '--notrace', default=False, help='do not print out a detailed trace', action='store_true', dest='notrace') +parser.add_option('-c', '--compute', default=False, help='compute answers for me', action='store_true', dest='solve') + +(options, args) = parser.parse_args() + +print 'ARG addresses', options.addresses +print 'ARG addressfile', options.addressfile +print 'ARG numaddrs', options.numaddrs +print 'ARG policy', options.policy +print 'ARG clockbits', options.clockbits +print 'ARG cachesize', options.cachesize +print 'ARG maxpage', options.maxpage +print 'ARG seed', options.seed +print 'ARG notrace', options.notrace +print '' + +addresses = str(options.addresses) +addressFile = str(options.addressfile) +numaddrs = int(options.numaddrs) +cachesize = int(options.cachesize) +seed = int(options.seed) +maxpage = int(options.maxpage) +policy = str(options.policy) +notrace = options.notrace +clockbits = int(options.clockbits) + +random.seed(seed) + +addrList = [] +if addressFile != '': + fd = open(addressFile) + for line in fd: + addrList.append(int(line)) + fd.close() +else: + if addresses == '-1': + # need to generate addresses + for i in range(0,numaddrs): + n = int(maxpage * random.random()) + addrList.append(n) + else: + addrList = addresses.split(',') + +if options.solve == False: + print 'Assuming a replacement policy of %s, and a cache of size %d pages,' % (policy, cachesize) + print 'figure out whether each of the following page references hit or miss' + print 'in the page cache.\n' + + for n in addrList: + print 'Access: %d Hit/Miss? State of Memory?' % int(n) + print '' + +else: + if notrace == False: + print 'Solving...\n' + + # init memory structure + count = 0 + memory = [] + hits = 0 + miss = 0 + + if policy == 'FIFO': + leftStr = 'FirstIn' + riteStr = 'Lastin ' + elif policy == 'LRU': + leftStr = 'LRU' + riteStr = 'MRU' + elif policy == 'MRU': + leftStr = 'LRU' + riteStr = 'MRU' + elif policy == 'OPT' or policy == 'RAND' or policy == 'UNOPT' or policy == 'CLOCK': + leftStr = 'Left ' + riteStr = 'Right' + else: + print 'Policy %s is not yet implemented' % policy + exit(1) + + # track reference bits for clock + ref = {} + + cdebug = False + + # need to generate addresses + addrIndex = 0 + for nStr in addrList: + # first, lookup + n = int(nStr) + try: + idx = memory.index(n) + hits = hits + 1 + if policy == 'LRU' or policy == 'MRU': + update = memory.remove(n) + memory.append(n) # puts it on MRU side + except: + idx = -1 + miss = miss + 1 + + victim = -1 + if idx == -1: + # miss, replace? + # print 'BUG count, cachesize:', count, cachesize + if count == cachesize: + # must replace + if policy == 'FIFO' or policy == 'LRU': + victim = memory.pop(0) + elif policy == 'MRU': + victim = memory.pop(count-1) + elif policy == 'RAND': + victim = memory.pop(int(random.random() * count)) + elif policy == 'CLOCK': + if cdebug: + print 'REFERENCE TO PAGE', n + print 'MEMORY ', memory + print 'REF (b)', ref + + # hack: for now, do random + # victim = memory.pop(int(random.random() * count)) + victim = -1 + while victim == -1: + page = memory[int(random.random() * count)] + if cdebug: + print ' scan page:', page, ref[page] + if ref[page] >= 1: + ref[page] -= 1 + else: + # this is our victim + victim = page + memory.remove(page) + break + + # remove old page's ref count + if page in memory: + assert('BROKEN') + del ref[victim] + if cdebug: + print 'VICTIM', page + print 'LEN', len(memory) + print 'MEM', memory + print 'REF (a)', ref + + elif policy == 'OPT': + maxReplace = -1 + replaceIdx = -1 + replacePage = -1 + # print 'OPT: access %d, memory %s' % (n, memory) + # print 'OPT: replace from FUTURE (%s)' % addrList[addrIndex+1:] + for pageIndex in range(0,count): + page = memory[pageIndex] + # now, have page 'page' at index 'pageIndex' in memory + whenReferenced = len(addrList) + # whenReferenced tells us when, in the future, this was referenced + for futureIdx in range(addrIndex+1,len(addrList)): + futurePage = int(addrList[futureIdx]) + if page == futurePage: + whenReferenced = futureIdx + break + # print 'OPT: page %d is referenced at %d' % (page, whenReferenced) + if whenReferenced >= maxReplace: + # print 'OPT: ??? updating maxReplace (%d %d %d)' % (replaceIdx, replacePage, maxReplace) + replaceIdx = pageIndex + replacePage = page + maxReplace = whenReferenced + # print 'OPT: --> updating maxReplace (%d %d %d)' % (replaceIdx, replacePage, maxReplace) + victim = memory.pop(replaceIdx) + # print 'OPT: replacing page %d (idx:%d) because I saw it in future at %d' % (victim, replaceIdx, whenReferenced) + elif policy == 'UNOPT': + minReplace = len(addrList) + 1 + replaceIdx = -1 + replacePage = -1 + for pageIndex in range(0,count): + page = memory[pageIndex] + # now, have page 'page' at index 'pageIndex' in memory + whenReferenced = len(addrList) + # whenReferenced tells us when, in the future, this was referenced + for futureIdx in range(addrIndex+1,len(addrList)): + futurePage = int(addrList[futureIdx]) + if page == futurePage: + whenReferenced = futureIdx + break + if whenReferenced < minReplace: + replaceIdx = pageIndex + replacePage = page + minReplace = whenReferenced + victim = memory.pop(replaceIdx) + else: + # miss, but no replacement needed (cache not full) + victim = -1 + count = count + 1 + + # now add to memory + memory.append(n) + if cdebug: + print 'LEN (a)', len(memory) + if victim != -1: + assert(victim not in memory) + + # after miss processing, update reference bit + if n not in ref: + ref[n] = 1 + else: + ref[n] += 1 + if ref[n] > clockbits: + ref[n] = clockbits + + if cdebug: + print 'REF (a)', ref + + if notrace == False: + print 'Access: %d %s %s -> %12s <- %s Replaced:%s [Hits:%d Misses:%d]' % (n, hfunc(idx), leftStr, memory, riteStr, vfunc(victim), hits, miss) + addrIndex = addrIndex + 1 + + print '' + print 'FINALSTATS hits %d misses %d hitrate %.2f' % (hits, miss, (100.0*float(hits))/(float(hits)+float(miss))) + print '' + + + + + + + + + + + + + diff --git a/related_info/ostep/ostep7-process-run.md b/related_info/ostep/ostep7-process-run.md new file mode 100644 index 0000000..eb68cdb --- /dev/null +++ b/related_info/ostep/ostep7-process-run.md @@ -0,0 +1,213 @@ + +This program, called process-run.py, allows you to see how the state of a +process state changes as it runs on a CPU. As described in the chapter, +processes can be in a few different states: + + RUNNING - the process is using the CPU right now + READY - the process could be using the CPU right now + but (alas) some other process is + WAITING - the process is waiting on I/O + (e.g., it issued a request to a disk) + DONE - the process is finished executing + +In this homework, we'll see how these process states change as a program +runs, and thus learn a little bit better how these things work. + +To run the program and get its options, do this: + +prompt> ./process-run.py -h + +If this doesn't work, type "python" before the command, like this: + +prompt> python process-run.py -h + +What you should see is this: + +Usage: process-run.py [options] + +Options: + -h, --help show this help message and exit + -s SEED, --seed=SEED the random seed + -l PROCESS_LIST, --processlist=PROCESS_LIST + a comma-separated list of processes to run, in the + form X1:Y1,X2:Y2,... where X is the number of + instructions that process should run, and Y the + chances (from 0 to 100) that an instruction will use + the CPU or issue an IO + -L IO_LENGTH, --iolength=IO_LENGTH + how long an IO takes + -S PROCESS_SWITCH_BEHAVIOR, --switch=PROCESS_SWITCH_BEHAVIOR + when to switch between processes: SWITCH_ON_IO, + SWITCH_ON_END + -I IO_DONE_BEHAVIOR, --iodone=IO_DONE_BEHAVIOR + type of behavior when IO ends: IO_RUN_LATER, + IO_RUN_IMMEDIATE + -c compute answers for me + -p, --printstats print statistics at end; only useful with -c flag + (otherwise stats are not printed) + +The most important option to understand is the PROCESS_LIST (as specified by +the -l or --processlist flags) which specifies exactly what each running +program (or "process") will do. A process consist of instructions, and each +instruction can just do one of two things: +- use the CPU +- issue an IO (and wait for it to complete) + +When a process uses the CPU (and does no IO at all), it should simply +alternate between RUNNING on the CPU or being READY to run. For example, here +is a simple run that just has one program being run, and that program only +uses the CPU (it does no IO). + +prompt> ./process-run.py -l 5:100 -p +Produce a trace of what would happen when you run these processes: +Process 0 + cpu + cpu + cpu + cpu + cpu + +Important behaviors: + System will switch when the current process is FINISHED or ISSUES AN IO + After IOs, the process issuing the IO will run LATER (when it is its turn) + +prompt> + +Here, the process we specified is "5:100" which means it should consist of 5 +instructions, and the chances that each instruction is a CPU instruction are +100%. + +You can see what happens to the process by using the -c flag, which computes the +answers for you: + +prompt> ./process-run.py -l 5:100 -c +Time PID: 0 CPU IOs + 1 RUN:cpu 1 + 2 RUN:cpu 1 + 3 RUN:cpu 1 + 4 RUN:cpu 1 + 5 RUN:cpu 1 + +This result is not too interesting: the process is simple in the RUN state and +then finishes, using the CPU the whole time and thus keeping the CPU busy the +entire run, and not doing any I/Os. + +Let's make it slightly more complex by running two processes: + +prompt> ./process-run.py -l 5:100,5:100 +Produce a trace of what would happen when you run these processes: +Process 0 + cpu + cpu + cpu + cpu + cpu + +Process 1 + cpu + cpu + cpu + cpu + cpu + +Important behaviors: + Scheduler will switch when the current process is FINISHED or ISSUES AN IO + After IOs, the process issuing the IO will run LATER (when it is its turn) + +In this case, two different processes run, each again just using the CPU. What +happens when the operating system runs them? Let's find out: + +prompt> ./process-run.py -l 5:100,5:100 -c +Time PID: 0 PID: 1 CPU IOs + 1 RUN:cpu READY 1 + 2 RUN:cpu READY 1 + 3 RUN:cpu READY 1 + 4 RUN:cpu READY 1 + 5 RUN:cpu READY 1 + 6 DONE RUN:cpu 1 + 7 DONE RUN:cpu 1 + 8 DONE RUN:cpu 1 + 9 DONE RUN:cpu 1 + 10 DONE RUN:cpu 1 + +As you can see above, first the process with "process ID" (or "PID") 0 runs, +while process 1 is READY to run but just waits until 0 is done. When 0 is +finished, it moves to the DONE state, while 1 runs. When 1 finishes, the trace +is done. + +Let's look at one more example before getting to some questions. In this +example, the process just issues I/O requests. + +prompt> ./process-run.py -l 3:0 +Produce a trace of what would happen when you run these processes: +Process 0 + io-start + io-start + io-start + +Important behaviors: + System will switch when the current process is FINISHED or ISSUES AN IO + After IOs, the process issuing the IO will run LATER (when it is its turn) + +What do you think the execution trace will look like? Let's find out: + +prompt> ./process-run.py -l 3:0 -c +Time PID: 0 CPU IOs + 1 RUN:io-start 1 + 2 WAITING 1 + 3 WAITING 1 + 4 WAITING 1 + 5 WAITING 1 + 6* RUN:io-start 1 + 7 WAITING 1 + 8 WAITING 1 + 9 WAITING 1 + 10 WAITING 1 + 11* RUN:io-start 1 + 12 WAITING 1 + 13 WAITING 1 + 14 WAITING 1 + 15 WAITING 1 + 16* DONE + +As you can see, the program just issues three I/Os. When each I/O is issued, +the process moves to a WAITING state, and while the device is busy servicing +the I/O, the CPU is idle. + +Let's print some stats (run the same command as above, but with the -p flag) +to see some overall behaviors: + +Stats: Total Time 16 +Stats: CPU Busy 3 (18.75%) +Stats: IO Busy 12 (75.00%) + +As you can see, the trace took 16 clock ticks to run, but the CPU was only +busy less than 20% of the time. The IO device, on the other hand, was quite +busy. In general, we'd like to keep all the devices busy, as that is a better +use of resources. + +There are a few other important flags: + -s SEED, --seed=SEED the random seed + this gives you way to create a bunch of different jobs randomly + + -L IO_LENGTH, --iolength=IO_LENGTH + this determines how long IOs take to complete (default is 5 ticks) + + -S PROCESS_SWITCH_BEHAVIOR, --switch=PROCESS_SWITCH_BEHAVIOR + when to switch between processes: SWITCH_ON_IO, SWITCH_ON_END + this determines when we switch to another process: + - SWITCH_ON_IO, the system will switch when a process issues an IO + - SWITCH_ON_END, the system will only switch when the current process is done + + -I IO_DONE_BEHAVIOR, --iodone=IO_DONE_BEHAVIOR + type of behavior when IO ends: IO_RUN_LATER, IO_RUN_IMMEDIATE + this determines when a process runs after it issues an IO: + - IO_RUN_IMMEDIATE: switch to this process right now + - IO_RUN_LATER: switch to this process when it is natural to + (e.g., depending on process-switching behavior) + +Now go answer the questions at the back of the chapter to learn more. + + + + diff --git a/related_info/ostep/ostep7-process-run.py b/related_info/ostep/ostep7-process-run.py new file mode 100755 index 0000000..c0fa175 --- /dev/null +++ b/related_info/ostep/ostep7-process-run.py @@ -0,0 +1,325 @@ +#! /usr/bin/env python + +import sys +from optparse import OptionParser +import random + +# process switch behavior +SCHED_SWITCH_ON_IO = 'SWITCH_ON_IO' +SCHED_SWITCH_ON_END = 'SWITCH_ON_END' + +# io finished behavior +IO_RUN_LATER = 'IO_RUN_LATER' +IO_RUN_IMMEDIATE = 'IO_RUN_IMMEDIATE' + +# process states +STATE_RUNNING = 'RUNNING' +STATE_READY = 'READY' +STATE_DONE = 'DONE' +STATE_WAIT = 'WAITING' + +# members of process structure +PROC_CODE = 'code_' +PROC_PC = 'pc_' +PROC_ID = 'pid_' +PROC_STATE = 'proc_state_' + +# things a process can do +DO_COMPUTE = 'cpu' +DO_IO = 'io' + + +class scheduler: + def __init__(self, process_switch_behavior, io_done_behavior, io_length): + # keep set of instructions for each of the processes + self.proc_info = {} + self.process_switch_behavior = process_switch_behavior + self.io_done_behavior = io_done_behavior + self.io_length = io_length + return + + def new_process(self): + proc_id = len(self.proc_info) + self.proc_info[proc_id] = {} + self.proc_info[proc_id][PROC_PC] = 0 + self.proc_info[proc_id][PROC_ID] = proc_id + self.proc_info[proc_id][PROC_CODE] = [] + self.proc_info[proc_id][PROC_STATE] = STATE_READY + return proc_id + + def load_file(self, progfile): + fd = open(progfile) + proc_id = self.new_process() + + for line in fd: + tmp = line.split() + if len(tmp) == 0: + continue + opcode = tmp[0] + if opcode == 'compute': + assert(len(tmp) == 2) + for i in range(int(tmp[1])): + self.proc_info[proc_id][PROC_CODE].append(DO_COMPUTE) + elif opcode == 'io': + assert(len(tmp) == 1) + self.proc_info[proc_id][PROC_CODE].append(DO_IO) + fd.close() + return + + def load(self, program_description): + proc_id = self.new_process() + tmp = program_description.split(':') + if len(tmp) != 2: + print 'Bad description (%s): Must be number ' + print ' where X is the number of instructions' + print ' and Y is the percent change that an instruction is CPU not IO' + exit(1) + + num_instructions, chance_cpu = int(tmp[0]), float(tmp[1])/100.0 + for i in range(num_instructions): + if random.random() < chance_cpu: + self.proc_info[proc_id][PROC_CODE].append(DO_COMPUTE) + else: + self.proc_info[proc_id][PROC_CODE].append(DO_IO) + return + + def move_to_ready(self, expected, pid=-1): + if pid == -1: + pid = self.curr_proc + assert(self.proc_info[pid][PROC_STATE] == expected) + self.proc_info[pid][PROC_STATE] = STATE_READY + return + + def move_to_wait(self, expected): + assert(self.proc_info[self.curr_proc][PROC_STATE] == expected) + self.proc_info[self.curr_proc][PROC_STATE] = STATE_WAIT + return + + def move_to_running(self, expected): + assert(self.proc_info[self.curr_proc][PROC_STATE] == expected) + self.proc_info[self.curr_proc][PROC_STATE] = STATE_RUNNING + return + + def move_to_done(self, expected): + assert(self.proc_info[self.curr_proc][PROC_STATE] == expected) + self.proc_info[self.curr_proc][PROC_STATE] = STATE_DONE + return + + def next_proc(self, pid=-1): + if pid != -1: + self.curr_proc = pid + self.move_to_running(STATE_READY) + return + for pid in range(self.curr_proc + 1, len(self.proc_info)): + if self.proc_info[pid][PROC_STATE] == STATE_READY: + self.curr_proc = pid + self.move_to_running(STATE_READY) + return + for pid in range(0, self.curr_proc + 1): + if self.proc_info[pid][PROC_STATE] == STATE_READY: + self.curr_proc = pid + self.move_to_running(STATE_READY) + return + return + + def get_num_processes(self): + return len(self.proc_info) + + def get_num_instructions(self, pid): + return len(self.proc_info[pid][PROC_CODE]) + + def get_instruction(self, pid, index): + return self.proc_info[pid][PROC_CODE][index] + + def get_num_active(self): + num_active = 0 + for pid in range(len(self.proc_info)): + if self.proc_info[pid][PROC_STATE] != STATE_DONE: + num_active += 1 + return num_active + + def get_num_runnable(self): + num_active = 0 + for pid in range(len(self.proc_info)): + if self.proc_info[pid][PROC_STATE] == STATE_READY or \ + self.proc_info[pid][PROC_STATE] == STATE_RUNNING: + num_active += 1 + return num_active + + def get_ios_in_flight(self, current_time): + num_in_flight = 0 + for pid in range(len(self.proc_info)): + for t in self.io_finish_times[pid]: + if t > current_time: + num_in_flight += 1 + return num_in_flight + + def check_for_switch(self): + return + + def space(self, num_columns): + for i in range(num_columns): + print '%10s' % ' ', + + def check_if_done(self): + if len(self.proc_info[self.curr_proc][PROC_CODE]) == 0: + if self.proc_info[self.curr_proc][PROC_STATE] == STATE_RUNNING: + self.move_to_done(STATE_RUNNING) + self.next_proc() + return + + def run(self): + clock_tick = 0 + + if len(self.proc_info) == 0: + return + + # track outstanding IOs, per process + self.io_finish_times = {} + for pid in range(len(self.proc_info)): + self.io_finish_times[pid] = [] + + # make first one active + self.curr_proc = 0 + self.move_to_running(STATE_READY) + + # OUTPUT: headers for each column + print '%s' % 'Time', + for pid in range(len(self.proc_info)): + print '%10s' % ('PID:%2d' % (pid)), + print '%10s' % 'CPU', + print '%10s' % 'IOs', + print '' + + # init statistics + io_busy = 0 + cpu_busy = 0 + + while self.get_num_active() > 0: + clock_tick += 1 + + # check for io finish + io_done = False + for pid in range(len(self.proc_info)): + if clock_tick in self.io_finish_times[pid]: + io_done = True + self.move_to_ready(STATE_WAIT, pid) + if self.io_done_behavior == IO_RUN_IMMEDIATE: + # IO_RUN_IMMEDIATE + if self.curr_proc != pid: + if self.proc_info[self.curr_proc][PROC_STATE] == STATE_RUNNING: + self.move_to_ready(STATE_RUNNING) + self.next_proc(pid) + else: + # IO_RUN_LATER + if self.process_switch_behavior == SCHED_SWITCH_ON_END: + # this means the process that issued the io should be run + self.next_proc(pid) + if self.get_num_runnable() == 1: + # this is the only thing to run: so run it + self.next_proc(pid) + self.check_if_done() + + # if current proc is RUNNING and has an instruction, execute it + instruction_to_execute = '' + if self.proc_info[self.curr_proc][PROC_STATE] == STATE_RUNNING and \ + len(self.proc_info[self.curr_proc][PROC_CODE]) > 0: + instruction_to_execute = self.proc_info[self.curr_proc][PROC_CODE].pop(0) + cpu_busy += 1 + + # OUTPUT: print what everyone is up to + if io_done: + print '%3d*' % clock_tick, + else: + print '%3d ' % clock_tick, + for pid in range(len(self.proc_info)): + if pid == self.curr_proc and instruction_to_execute != '': + print '%10s' % ('RUN:'+instruction_to_execute), + else: + print '%10s' % (self.proc_info[pid][PROC_STATE]), + if instruction_to_execute == '': + print '%10s' % ' ', + else: + print '%10s' % 1, + num_outstanding = self.get_ios_in_flight(clock_tick) + if num_outstanding > 0: + print '%10s' % str(num_outstanding), + io_busy += 1 + else: + print '%10s' % ' ', + print '' + + # if this is an IO instruction, switch to waiting state + # and add an io completion in the future + if instruction_to_execute == DO_IO: + self.move_to_wait(STATE_RUNNING) + self.io_finish_times[self.curr_proc].append(clock_tick + self.io_length) + if self.process_switch_behavior == SCHED_SWITCH_ON_IO: + self.next_proc() + + # ENDCASE: check if currently running thing is out of instructions + self.check_if_done() + return (cpu_busy, io_busy, clock_tick) + +# +# PARSE ARGUMENTS +# + +parser = OptionParser() +parser.add_option('-s', '--seed', default=0, help='the random seed', action='store', type='int', dest='seed') +parser.add_option('-l', '--processlist', default='', + help='a comma-separated list of processes to run, in the form X1:Y1,X2:Y2,... where X is the number of instructions that process should run, and Y the chances (from 0 to 100) that an instruction will use the CPU or issue an IO', + action='store', type='string', dest='process_list') +parser.add_option('-L', '--iolength', default=5, help='how long an IO takes', action='store', type='int', dest='io_length') +parser.add_option('-S', '--switch', default='SWITCH_ON_IO', + help='when to switch between processes: SWITCH_ON_IO, SWITCH_ON_END', + action='store', type='string', dest='process_switch_behavior') +parser.add_option('-I', '--iodone', default='IO_RUN_LATER', + help='type of behavior when IO ends: IO_RUN_LATER, IO_RUN_IMMEDIATE', + action='store', type='string', dest='io_done_behavior') +parser.add_option('-c', help='compute answers for me', action='store_true', default=False, dest='solve') +parser.add_option('-p', '--printstats', help='print statistics at end; only useful with -c flag (otherwise stats are not printed)', action='store_true', default=False, dest='print_stats') +(options, args) = parser.parse_args() + +random.seed(options.seed) + +assert(options.process_switch_behavior == SCHED_SWITCH_ON_IO or \ + options.process_switch_behavior == SCHED_SWITCH_ON_END) +assert(options.io_done_behavior == IO_RUN_IMMEDIATE or \ + options.io_done_behavior == IO_RUN_LATER) + +s = scheduler(options.process_switch_behavior, options.io_done_behavior, options.io_length) + +# example process description (10:100,10:100) +for p in options.process_list.split(','): + s.load(p) + +if options.solve == False: + print 'Produce a trace of what would happen when you run these processes:' + for pid in range(s.get_num_processes()): + print 'Process %d' % pid + for inst in range(s.get_num_instructions(pid)): + print ' %s' % s.get_instruction(pid, inst) + print '' + print 'Important behaviors:' + print ' System will switch when', + if options.process_switch_behavior == SCHED_SWITCH_ON_IO: + print 'the current process is FINISHED or ISSUES AN IO' + else: + print 'the current process is FINISHED' + print ' After IOs, the process issuing the IO will', + if options.io_done_behavior == IO_RUN_IMMEDIATE: + print 'run IMMEDIATELY' + else: + print 'run LATER (when it is its turn)' + print '' + exit(0) + +(cpu_busy, io_busy, clock_tick) = s.run() + +if options.print_stats: + print '' + print 'Stats: Total Time %d' % clock_tick + print 'Stats: CPU Busy %d (%.2f%%)' % (cpu_busy, 100.0 * float(cpu_busy)/clock_tick) + print 'Stats: IO Busy %d (%.2f%%)' % (io_busy, 100.0 * float(io_busy)/clock_tick) + print '' diff --git a/related_info/ostep/ostep8-scheduler.md b/related_info/ostep/ostep8-scheduler.md new file mode 100644 index 0000000..b362cc1 --- /dev/null +++ b/related_info/ostep/ostep8-scheduler.md @@ -0,0 +1,116 @@ + +This program, scheduler.py, allows you to see how different schedulers perform +under scheduling metrics such as response time, turnaround time, and total +wait time. Three schedulers are "implemented": FIFO, SJF, and RR. + +There are two steps to running the program. + +First, run without the -c flag: this shows you what problem to solve without +revealing the answers. For example, if you want to compute response, +turnaround, and wait for three jobs using the FIFO policy, run this: + + ./scheduler.py -p FIFO -j 3 -s 100 + +If that doesn't work, try this: + python ./scheduler.py -p FIFO -j 3 -s 100 + +This specifies the FIFO policy with three jobs, and, importantly, a specific +random seed of 100. If you want to see the solution for this exact problem, +you have to specify this exact same random seed again. Let's run it and see +what happens. This is what you should see: + +prompt> ./scheduler.py -p FIFO -j 3 -s 100 +ARG policy FIFO +ARG jobs 3 +ARG maxlen 10 +ARG seed 100 + +Here is the job list, with the run time of each job: + Job 0 (length = 1) + Job 1 (length = 4) + Job 2 (length = 7) + +Compute the turnaround time, response time, and wait time for each job. When +you are done, run this program again, with the same arguments, but with -c, +which will thus provide you with the answers. You can use -s or +your own job list (-l 10,15,20 for example) to generate different problems for +yourself. + +As you can see from this example, three jobs are generated: job 0 of length 1, +job 1 of length 4, and job 2 of length 7. As the program states, you can now +use this to compute some statistics and see if you have a grip on the basic +concepts. + +Once you are done, you can use the same program to "solve" the problem and see +if you did your work correctly. To do so, use the "-c" flag. The output: + +prompt> ./scheduler.py -p FIFO -j 3 -s 100 -c +ARG policy FIFO +ARG jobs 3 +ARG maxlen 10 +ARG seed 100 + +Here is the job list, with the run time of each job: + Job 0 (length = 1) + Job 1 (length = 4) + Job 2 (length = 7) + +** Solutions ** + +Execution trace: + [time 0] Run job 0 for 1.00 secs (DONE) + [time 1] Run job 1 for 4.00 secs (DONE) + [time 5] Run job 2 for 7.00 secs (DONE) + +Final statistics: + Job 0 -- Response: 0.00 Turnaround 1.00 Wait 0.00 + Job 1 -- Response: 1.00 Turnaround 5.00 Wait 1.00 + Job 2 -- Response: 5.00 Turnaround 12.00 Wait 5.00 + + Average -- Response: 2.00 Turnaround 6.00 Wait 2.00 + +As you can see from the figure, the -c flag shows you what happened. Job 0 ran +first for 1 second, Job 1 ran second for 4, and then Job 2 ran for 7 +seconds. Not too hard; it is FIFO, after all! The execution trace shows these +results. + +The final statistics are useful too: they compute the "response time" (the +time a job spends waiting after arrival before first running), the "turnaround +time" (the time it took to complete the job since first arrival), and the +total "wait time" (any time spent ready but not running). The stats are shown +per job and then as an average across all jobs. Of course, you should have +computed these things all before running with the "-c" flag! + +If you want to try the same type of problem but with different inputs, try +changing the number of jobs or the random seed or both. Different random seeds +basically give you a way to generate an infinite number of different problems +for yourself, and the "-c" flag lets you check your own work. Keep doing this +until you feel like you really understand the concepts. + +One other useful flag is "-l" (that's a lower-case L), which lets you specify +the exact jobs you wish to see scheduled. For example, if you want to find out +how SJF would perform with three jobs of lengths 5, 10, and 15, you can run: + +prompt> ./scheduler.py -p SJF -l 5,10,15 +ARG policy SJF +ARG jlist 5,10,15 + +Here is the job list, with the run time of each job: + Job 0 (length = 5.0) + Job 1 (length = 10.0) + Job 2 (length = 15.0) +... + +And then you can use -c to solve it again. Note that when you specify the +exact jobs, there is no need to specify a random seed or the number of jobs: +the jobs lengths are taken from your comma-separated list. + +Of course, more interesting things happen when you use SJF (shortest-job +first) or even RR (round robin) schedulers. Try them and see! + +And you can always run + + ./scheduler.py -h + +to get a complete list of flags and options (including options such as setting +the time quantum for the RR scheduler). diff --git a/related_info/ostep/ostep8-scheduler.py b/related_info/ostep/ostep8-scheduler.py new file mode 100755 index 0000000..fdfd19c --- /dev/null +++ b/related_info/ostep/ostep8-scheduler.py @@ -0,0 +1,155 @@ +#! /usr/bin/env python + +import sys +from optparse import OptionParser +import random + +parser = OptionParser() +parser.add_option("-s", "--seed", default=0, help="the random seed", + action="store", type="int", dest="seed") +parser.add_option("-j", "--jobs", default=3, help="number of jobs in the system", + action="store", type="int", dest="jobs") +parser.add_option("-l", "--jlist", default="", help="instead of random jobs, provide a comma-separated list of run times", + action="store", type="string", dest="jlist") +parser.add_option("-m", "--maxlen", default=10, help="max length of job", + action="store", type="int", dest="maxlen") +parser.add_option("-p", "--policy", default="FIFO", help="sched policy to use: SJF, FIFO, RR", + action="store", type="string", dest="policy") +parser.add_option("-q", "--quantum", help="length of time slice for RR policy", default=1, + action="store", type="int", dest="quantum") +parser.add_option("-c", help="compute answers for me", action="store_true", default=False, dest="solve") + +(options, args) = parser.parse_args() + +random.seed(options.seed) + +print 'ARG policy', options.policy +if options.jlist == '': + print 'ARG jobs', options.jobs + print 'ARG maxlen', options.maxlen + print 'ARG seed', options.seed +else: + print 'ARG jlist', options.jlist + +print '' + +print 'Here is the job list, with the run time of each job: ' + +import operator + +joblist = [] +if options.jlist == '': + for jobnum in range(0,options.jobs): + runtime = int(options.maxlen * random.random()) + 1 + joblist.append([jobnum, runtime]) + print ' Job', jobnum, '( length = ' + str(runtime) + ' )' +else: + jobnum = 0 + for runtime in options.jlist.split(','): + joblist.append([jobnum, float(runtime)]) + jobnum += 1 + for job in joblist: + print ' Job', job[0], '( length = ' + str(job[1]) + ' )' +print '\n' + +if options.solve == True: + print '** Solutions **\n' + if options.policy == 'SJF': + joblist = sorted(joblist, key=operator.itemgetter(1)) + options.policy = 'FIFO' + + if options.policy == 'FIFO': + thetime = 0 + print 'Execution trace:' + for job in joblist: + print ' [ time %3d ] Run job %d for %.2f secs ( DONE at %.2f )' % (thetime, job[0], job[1], thetime + job[1]) + thetime += job[1] + + print '\nFinal statistics:' + t = 0.0 + count = 0 + turnaroundSum = 0.0 + waitSum = 0.0 + responseSum = 0.0 + for tmp in joblist: + jobnum = tmp[0] + runtime = tmp[1] + + response = t + turnaround = t + runtime + wait = t + print ' Job %3d -- Response: %3.2f Turnaround %3.2f Wait %3.2f' % (jobnum, response, turnaround, wait) + responseSum += response + turnaroundSum += turnaround + waitSum += wait + t += runtime + count = count + 1 + print '\n Average -- Response: %3.2f Turnaround %3.2f Wait %3.2f\n' % (responseSum/count, turnaroundSum/count, waitSum/count) + + if options.policy == 'RR': + print 'Execution trace:' + turnaround = {} + response = {} + lastran = {} + wait = {} + quantum = float(options.quantum) + jobcount = len(joblist) + for i in range(0,jobcount): + lastran[i] = 0.0 + wait[i] = 0.0 + turnaround[i] = 0.0 + response[i] = -1 + + runlist = [] + for e in joblist: + runlist.append(e) + + thetime = 0.0 + while jobcount > 0: + # print '%d jobs remaining' % jobcount + job = runlist.pop(0) + jobnum = job[0] + runtime = float(job[1]) + if response[jobnum] == -1: + response[jobnum] = thetime + currwait = thetime - lastran[jobnum] + wait[jobnum] += currwait + if runtime > quantum: + runtime -= quantum + ranfor = quantum + print ' [ time %3d ] Run job %3d for %.2f secs' % (thetime, jobnum, ranfor) + runlist.append([jobnum, runtime]) + else: + ranfor = runtime; + print ' [ time %3d ] Run job %3d for %.2f secs ( DONE at %.2f )' % (thetime, jobnum, ranfor, thetime + ranfor) + turnaround[jobnum] = thetime + ranfor + jobcount -= 1 + thetime += ranfor + lastran[jobnum] = thetime + + print '\nFinal statistics:' + turnaroundSum = 0.0 + waitSum = 0.0 + responseSum = 0.0 + for i in range(0,len(joblist)): + turnaroundSum += turnaround[i] + responseSum += response[i] + waitSum += wait[i] + print ' Job %3d -- Response: %3.2f Turnaround %3.2f Wait %3.2f' % (i, response[i], turnaround[i], wait[i]) + count = len(joblist) + + print '\n Average -- Response: %3.2f Turnaround %3.2f Wait %3.2f\n' % (responseSum/count, turnaroundSum/count, waitSum/count) + + if options.policy != 'FIFO' and options.policy != 'SJF' and options.policy != 'RR': + print 'Error: Policy', options.policy, 'is not available.' + sys.exit(0) +else: + print 'Compute the turnaround time, response time, and wait time for each job.' + print 'When you are done, run this program again, with the same arguments,' + print 'but with -c, which will thus provide you with the answers. You can use' + print '-s or your own job list (-l 10,15,20 for example)' + print 'to generate different problems for yourself.' + print '' + + + diff --git a/related_info/ostep/ostep9-mlfq.md b/related_info/ostep/ostep9-mlfq.md new file mode 100644 index 0000000..57bd668 --- /dev/null +++ b/related_info/ostep/ostep9-mlfq.md @@ -0,0 +1,174 @@ + +This program, mlfq.py, allows you to see how the MLFQ scheduler +presented in this chapter behaves. As before, you can use this to generate +problems for yourself using random seeds, or use it to construct a +carefully-designed experiment to see how MLFQ works under different +circumstances. To run the program, type: + +prompt> ./mlfq.py + +Use the help flag (-h) to see the options: + +Usage: mlfq.py [options] +Options: + -h, --help show this help message and exit + -s SEED, --seed=SEED the random seed + -n NUMQUEUES, --numQueues=NUMQUEUES + number of queues in MLFQ (if not using -Q) + -q QUANTUM, --quantum=QUANTUM + length of time slice (if not using -Q) + -Q QUANTUMLIST, --quantumList=QUANTUMLIST + length of time slice per queue level, + specified as x,y,z,... where x is the + quantum length for the highest-priority + queue, y the next highest, and so forth + -j NUMJOBS, --numJobs=NUMJOBS + number of jobs in the system + -m MAXLEN, --maxlen=MAXLEN + max run-time of a job (if random) + -M MAXIO, --maxio=MAXIO + max I/O frequency of a job (if random) + -B BOOST, --boost=BOOST + how often to boost the priority of all + jobs back to high priority (0 means never) + -i IOTIME, --iotime=IOTIME + how long an I/O should last (fixed constant) + -S, --stay reset and stay at same priority level + when issuing I/O + -l JLIST, --jlist=JLIST + a comma-separated list of jobs to run, + in the form x1,y1,z1:x2,y2,z2:... where + x is start time, y is run time, and z + is how often the job issues an I/O request + -c compute answers for me +] + +There are a few different ways to use the simulator. One way is to generate +some random jobs and see if you can figure out how they will behave given the +MLFQ scheduler. For example, if you wanted to create a randomly-generated +three-job workload, you would simply type: + +prompt> ./mlfq.py -j 3 + +What you would then see is the specific problem definition: + +Here is the list of inputs: +OPTIONS jobs 3 +OPTIONS queues 3 +OPTIONS quantum length for queue 2 is 10 +OPTIONS quantum length for queue 1 is 10 +OPTIONS quantum length for queue 0 is 10 +OPTIONS boost 0 +OPTIONS ioTime 0 +OPTIONS stayAfterIO False + + +For each job, three defining characteristics are given: + startTime : at what time does the job enter the system + runTime : the total CPU time needed by the job to finish + ioFreq : every ioFreq time units, the job issues an I/O + (the I/O takes ioTime units to complete) + +Job List: + Job 0: startTime 0 - runTime 84 - ioFreq 7 + Job 1: startTime 0 - runTime 42 - ioFreq 2 + Job 2: startTime 0 - runTime 51 - ioFreq 4 + +Compute the execution trace for the given workloads. +If you would like, also compute the response and turnaround +times for each of the jobs. + +Use the -c flag to get the exact results when you are finished. + +This generates a random workload of three jobs (as specified), on the default +number of queues with a number of default settings. If you run again with the +solve flag on (-c), you'll see the same print out as above, plus the +following: + +Execution Trace: + +[time 0] JOB BEGINS by JOB 0 +[time 0] JOB BEGINS by JOB 1 +[time 0] JOB BEGINS by JOB 2 +[time 0] Run JOB 0 at PRI 2 [TICKSLEFT 9 RUNTIME 84 TIMELEFT 83] +[time 1] Run JOB 0 at PRI 2 [TICKSLEFT 8 RUNTIME 84 TIMELEFT 82] +[time 2] Run JOB 0 at PRI 2 [TICKSLEFT 7 RUNTIME 84 TIMELEFT 81] +[time 3] Run JOB 0 at PRI 2 [TICKSLEFT 6 RUNTIME 84 TIMELEFT 80] +[time 4] Run JOB 0 at PRI 2 [TICKSLEFT 5 RUNTIME 84 TIMELEFT 79] +[time 5] Run JOB 0 at PRI 2 [TICKSLEFT 4 RUNTIME 84 TIMELEFT 78] +[time 6] Run JOB 0 at PRI 2 [TICKSLEFT 3 RUNTIME 84 TIMELEFT 77] +[time 7] IO_START by JOB 0 +[time 7] Run JOB 1 at PRI 2 [TICKSLEFT 9 RUNTIME 42 TIMELEFT 41] +[time 8] Run JOB 1 at PRI 2 [TICKSLEFT 8 RUNTIME 42 TIMELEFT 40] +[time 9] IO_START by JOB 1 + +... + +Final statistics: + Job 0: startTime 0 - response 0 - turnaround 175 + Job 1: startTime 0 - response 7 - turnaround 191 + Job 2: startTime 0 - response 9 - turnaround 168 + + Avg 2: startTime n/a - response 5.33 - turnaround 178.00 +] + +The trace shows exactly, on a millisecond-by-millisecond time scale, what the +scheduler decided to do. In this example, it begins by running Job 0 for 7 ms +until Job 0 issues an I/O; this is entirely predictable, as Job 0's I/O +frequency is set to 7 ms, meaning that every 7 ms it runs, it will issue an +I/O and wait for it to complete before continuing. At that point, the +scheduler switches to Job 1, which only runs 2 ms before issuing an I/O. +The scheduler prints the entire execution trace in this manner, and +finally also computes the response and turnaround times for each job +as well as an average. + +You can also control various other aspects of the simulation. For example, you +can specify how many queues you'd like to have in the system (-n) and what the +quantum length should be for all of those queues (-q); if you want even more +control and varied quanta length per queue, you can instead specify the length +of the quantum for each queue with -Q, e.g., -Q 10,20,30] simulates a +scheduler with three queues, with the highest-priority queue having a 10-ms +time slice, the next-highest a 20-ms time-slice, and the low-priority queue a +30-ms time slice. + +If you are randomly generating jobs, you can also control how long they might +run for (-m), or how often they generate I/O (-M). If you, however, want more +control over the exact characteristics of the jobs running in the system, you +can use -l (lower-case L) or --jlist, which allows you to specify the exact +set of jobs you wish to simulate. The list is of the form: +x1,y1,z1:x2,y2,z2:... where x is the start time of the job, y is the run time +(i.e., how much CPU time it needs), and z the I/O frequency (i.e., after +running z ms, the job issues an I/O; if z is 0, no I/Os are issued). + +For example, if you wanted to recreate the example in Figure 8.3 +you would specify a job list as follows: + +prompt> ./mlfq.py --jlist 0,180,0:100,20,0 -Q 10,10,10 + +Running the simulator in this way creates a three-level MLFQ, with each level +having a 10-ms time slice. Two jobs are created: Job 0 which starts at time 0, +runs for 180 ms total, and never issues an I/O; Job 1 starts at 100 ms, needs +only 20 ms of CPU time to complete, and also never issues I/Os. + +Finally, there are three more parameters of interest. The -B flag, if set to a +non-zero value, boosts all jobs to the highest-priority queue every N +milliseconds, when invoked as such: + prompt> ./mlfq.py -B N +The scheduler uses this feature to avoid starvation as discussed in the +chapter. However, it is off by default. + +The -S flag invokes older Rules 4a and 4b, which means that if a job issues an +I/O before completing its time slice, it will return to that same priority +queue when it resumes execution, with its full time-slice intact. This +enables gaming of the scheduler. + +Finally, you can easily change how long an I/O lasts by using the -i flag. By +default in this simplistic model, each I/O takes a fixed amount of time of 5 +milliseconds or whatever you set it to with this flag. + +You can also play around with whether jobs that just complete an I/O are moved +to the head of the queue they are in or to the back, with the -I flag. Check +it out. + + + diff --git a/related_info/ostep/ostep9-mlfq.py b/related_info/ostep/ostep9-mlfq.py new file mode 100755 index 0000000..5574ded --- /dev/null +++ b/related_info/ostep/ostep9-mlfq.py @@ -0,0 +1,326 @@ +#! /usr/bin/env python + +import sys +from optparse import OptionParser +import random + +# finds the highest nonempty queue +# -1 if they are all empty +def FindQueue(): + q = hiQueue + while q > 0: + if len(queue[q]) > 0: + return q + q -= 1 + if len(queue[0]) > 0: + return 0 + return -1 + +def LowerQueue(currJob, currQueue, issuedIO): + if currQueue > 0: + # in this case, have to change the priority of the job + job[currJob]['currPri'] = currQueue - 1 + if issuedIO == False: + queue[currQueue-1].append(currJob) + job[currJob]['ticksLeft'] = quantum[currQueue-1] + else: + if issuedIO == False: + queue[currQueue].append(currJob) + job[currJob]['ticksLeft'] = quantum[currQueue] + +def Abort(str): + sys.stderr.write(str + '\n') + exit(1) + + +# +# PARSE ARGUMENTS +# + +parser = OptionParser() +parser.add_option('-s', '--seed', default=0, help='the random seed', + action='store', type='int', dest='seed') +parser.add_option('-n', '--numQueues', help='number of queues in MLFQ (if not using -Q)', default=3, + action='store', type='int', dest='numQueues') +parser.add_option('-q', '--quantum', help='length of time slice (if not using -Q)', default=10, + action='store', type='int', dest='quantum') +parser.add_option('-Q', '--quantumList', help='length of time slice per queue level, specified as x,y,z,... where x is the quantum length for the highest priority queue, y the next highest, and so forth', + default='', action='store', type='string', dest='quantumList') +parser.add_option('-j', '--numJobs', default=3, help='number of jobs in the system', + action='store', type='int', dest='numJobs') +parser.add_option('-m', '--maxlen', default=100, help='max run-time of a job (if randomly generating)', + action='store', type='int', dest='maxlen') +parser.add_option('-M', '--maxio', default=10, help='max I/O frequency of a job (if randomly generating)', + action='store', type='int', dest='maxio') +parser.add_option('-B', '--boost', default=0, help='how often to boost the priority of all jobs back to high priority', + action='store', type='int', dest='boost') +parser.add_option('-i', '--iotime', default=5, help='how long an I/O should last (fixed constant)', + action='store', type='int', dest='ioTime') +parser.add_option('-S', '--stay', default=False, help='reset and stay at same priority level when issuing I/O', + action='store_true', dest='stay') +parser.add_option('-I', '--iobump', default=False, help='if specified, jobs that finished I/O move immediately to front of current queue', + action='store_true', dest='iobump') +parser.add_option('-l', '--jlist', default='', help='a comma-separated list of jobs to run, in the form x1,y1,z1:x2,y2,z2:... where x is start time, y is run time, and z is how often the job issues an I/O request', + action='store', type='string', dest='jlist') +parser.add_option('-c', help='compute answers for me', action='store_true', default=False, dest='solve') + +(options, args) = parser.parse_args() + +random.seed(options.seed) + + +# MLFQ: How Many Queues +numQueues = options.numQueues + +quantum = {} +if options.quantumList != '': + # instead, extract number of queues and their time slic + quantumLengths = options.quantumList.split(',') + numQueues = len(quantumLengths) + qc = numQueues - 1 + for i in range(numQueues): + quantum[qc] = int(quantumLengths[i]) + qc -= 1 +else: + for i in range(numQueues): + quantum[i] = int(options.quantum) + +hiQueue = numQueues - 1 + +# MLFQ: I/O Model +# the time for each IO: not great to have a single fixed time but... +ioTime = int(options.ioTime) + +# This tracks when IOs and other interrupts are complete +ioDone = {} + +# This stores all info about the jobs +job = {} + +# seed the random generator +random.seed(options.seed) + +# jlist 'startTime,runTime,ioFreq:startTime,runTime,ioFreq:...' +jobCnt = 0 +if options.jlist != '': + allJobs = options.jlist.split(':') + for j in allJobs: + jobInfo = j.split(',') + if len(jobInfo) != 3: + sys.stderr.write('Badly formatted job string. Should be x1,y1,z1:x2,y2,z2:...\n') + sys.stderr.write('where x is the startTime, y is the runTime, and z is the I/O frequency.\n') + exit(1) + assert(len(jobInfo) == 3) + startTime = int(jobInfo[0]) + runTime = int(jobInfo[1]) + ioFreq = int(jobInfo[2]) + job[jobCnt] = {'currPri':hiQueue, 'ticksLeft':quantum[hiQueue], 'startTime':startTime, + 'runTime':runTime, 'timeLeft':runTime, 'ioFreq':ioFreq, 'doingIO':False, + 'firstRun':-1} + if startTime not in ioDone: + ioDone[startTime] = [] + ioDone[startTime].append((jobCnt, 'JOB BEGINS')) + jobCnt += 1 +else: + # do something random + for j in range(options.numJobs): + startTime = 0 + runTime = int(random.random() * options.maxlen) + ioFreq = int(random.random() * options.maxio) + + job[jobCnt] = {'currPri':hiQueue, 'ticksLeft':quantum[hiQueue], 'startTime':startTime, + 'runTime':runTime, 'timeLeft':runTime, 'ioFreq':ioFreq, 'doingIO':False, + 'firstRun':-1} + if startTime not in ioDone: + ioDone[startTime] = [] + ioDone[startTime].append((jobCnt, 'JOB BEGINS')) + jobCnt += 1 + + +numJobs = len(job) + +print 'Here is the list of inputs:' +print 'OPTIONS jobs', numJobs +print 'OPTIONS queues', numQueues +for i in range(len(quantum)-1,-1,-1): + print 'OPTIONS quantum length for queue %2d is %3d' % (i, quantum[i]) +print 'OPTIONS boost', options.boost +print 'OPTIONS ioTime', options.ioTime +print 'OPTIONS stayAfterIO', options.stay +print 'OPTIONS iobump', options.iobump + +print '\n' +print 'For each job, three defining characteristics are given:' +print ' startTime : at what time does the job enter the system' +print ' runTime : the total CPU time needed by the job to finish' +print ' ioFreq : every ioFreq time units, the job issues an I/O' +print ' (the I/O takes ioTime units to complete)\n' + +print 'Job List:' +for i in range(numJobs): + print ' Job %2d: startTime %3d - runTime %3d - ioFreq %3d' % (i, job[i]['startTime'], + job[i]['runTime'], job[i]['ioFreq']) +print '' + +if options.solve == False: + print 'Compute the execution trace for the given workloads.' + print 'If you would like, also compute the response and turnaround' + print 'times for each of the jobs.' + print '' + print 'Use the -c flag to get the exact results when you are finished.\n' + exit(0) + +# initialize the MLFQ queues +queue = {} +for q in range(numQueues): + queue[q] = [] + +# TIME IS CENTRAL +currTime = 0 + +# use these to know when we're finished +totalJobs = len(job) +finishedJobs = 0 + +print '\nExecution Trace:\n' + +while finishedJobs < totalJobs: + # find highest priority job + # run it until either + # (a) the job uses up its time quantum + # (b) the job performs an I/O + + # check for priority boost + if options.boost > 0 and currTime != 0: + if currTime % options.boost == 0: + print '[ time %d ] BOOST ( every %d )' % (currTime, options.boost) + # remove all jobs from queues (except high queue) + for q in range(numQueues-1): + for j in queue[q]: + if job[j]['doingIO'] == False: + queue[hiQueue].append(j) + queue[q] = [] + # print 'BOOST: QUEUES look like:', queue + + # change priority to high priority + # reset number of ticks left for all jobs (XXX just for lower jobs?) + # add to highest run queue (if not doing I/O) + for j in range(numJobs): + # print '-> Boost %d (timeLeft %d)' % (j, job[j]['timeLeft']) + if job[j]['timeLeft'] > 0: + # print '-> FinalBoost %d (timeLeft %d)' % (j, job[j]['timeLeft']) + job[j]['currPri'] = hiQueue + job[j]['ticksLeft'] = quantum[hiQueue] + # print 'BOOST END: QUEUES look like:', queue + + # check for any I/Os done + if currTime in ioDone: + for (j, type) in ioDone[currTime]: + q = job[j]['currPri'] + job[j]['doingIO'] = False + print '[ time %d ] %s by JOB %d' % (currTime, type, j) + if options.iobump == False: + queue[q].append(j) + else: + queue[q].insert(0, j) + + # now find the highest priority job + currQueue = FindQueue() + if currQueue == -1: + print '[ time %d ] IDLE' % (currTime) + currTime += 1 + continue + #print 'FOUND QUEUE: %d' % currQueue + #print 'ALL QUEUES:', queue + + # there was at least one runnable job, and hence ... + currJob = queue[currQueue][0] + if job[currJob]['currPri'] != currQueue: + Abort('currPri[%d] does not match currQueue[%d]' % (job[currJob]['currPri'], currQueue)) + + job[currJob]['timeLeft'] -= 1 + job[currJob]['ticksLeft'] -= 1 + + if job[currJob]['firstRun'] == -1: + job[currJob]['firstRun'] = currTime + + runTime = job[currJob]['runTime'] + ioFreq = job[currJob]['ioFreq'] + ticksLeft = job[currJob]['ticksLeft'] + timeLeft = job[currJob]['timeLeft'] + + print '[ time %d ] Run JOB %d at PRIORITY %d [ TICKSLEFT %d RUNTIME %d TIMELEFT %d ]' % (currTime, currJob, currQueue, ticksLeft, runTime, timeLeft) + + if timeLeft < 0: + Abort('Error: should never have less than 0 time left to run') + + + # UPDATE TIME + currTime += 1 + + # CHECK FOR JOB ENDING + if timeLeft == 0: + print '[ time %d ] FINISHED JOB %d' % (currTime, currJob) + finishedJobs += 1 + job[currJob]['endTime'] = currTime + # print 'BEFORE POP', queue + done = queue[currQueue].pop(0) + # print 'AFTER POP', queue + assert(done == currJob) + continue + + # CHECK FOR IO + issuedIO = False + if ioFreq > 0 and (((runTime - timeLeft) % ioFreq) == 0): + # time for an IO! + print '[ time %d ] IO_START by JOB %d' % (currTime, currJob) + issuedIO = True + desched = queue[currQueue].pop(0) + assert(desched == currJob) + job[currJob]['doingIO'] = True + # this does the bad rule -- reset your tick counter if you stay at the same level + if options.stay == True: + job[currJob]['ticksLeft'] = quantum[currQueue] + # add to IO Queue: but which queue? + futureTime = currTime + ioTime + if futureTime not in ioDone: + ioDone[futureTime] = [] + ioDone[futureTime].append((currJob, 'IO_DONE')) + # print 'NEW IO EVENT at ', futureTime, ' is ', ioDone[futureTime] + + # CHECK FOR QUANTUM ENDING AT THIS LEVEL + if ticksLeft == 0: + # print '--> DESCHEDULE %d' % currJob + if issuedIO == False: + # print '--> BUT IO HAS NOT BEEN ISSUED (therefor pop from queue)' + desched = queue[currQueue].pop(0) + assert(desched == currJob) + # move down one queue! (unless lowest queue) + LowerQueue(currJob, currQueue, issuedIO) + + +# print out statistics +print '' +print 'Final statistics:' +responseSum = 0 +turnaroundSum = 0 +for i in range(numJobs): + response = job[i]['firstRun'] - job[i]['startTime'] + turnaround = job[i]['endTime'] - job[i]['startTime'] + print ' Job %2d: startTime %3d - response %3d - turnaround %3d' % (i, job[i]['startTime'], + response, turnaround) + responseSum += response + turnaroundSum += turnaround + +print '\n Avg %2d: startTime n/a - response %.2f - turnaround %.2f' % (i, + float(responseSum)/numJobs, + float(turnaroundSum)/numJobs) + +print '\n' + + + + + + +