# HG changeset patch # User Steve Losh # Date 1475844516 0 # Node ID e7bc59b9ebda560f51f5221b73d8063a3fdbb02a # Parent bbf39c61e3fef07468d1e2a238ce51347d379f3f Switch to Hugo diff -r bbf39c61e3fe -r e7bc59b9ebda DESIGN.mdown --- a/DESIGN.mdown Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,13 +0,0 @@ -Some notes on the design, mostly for myself. - -Images -====== - -Full images for posts should be 580 pixels wide (or less) and a multiple of 25 -pixels tall. - -Headers -======= - -Blog and project content block should use h2 and lower. h1 elements are -reserved for the layout. diff -r bbf39c61e3fe -r e7bc59b9ebda LICENSE --- a/LICENSE Tue Sep 20 15:23:05 2016 +0000 +++ b/LICENSE Fri Oct 07 12:48:36 2016 +0000 @@ -1,4 +1,4 @@ -Copyright (c) 2008-2010 Steve Losh +Copyright (c) 2016 Steve Losh Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -16,4 +16,4 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. \ No newline at end of file +THE SOFTWARE. diff -r bbf39c61e3fe -r e7bc59b9ebda Makefile --- a/Makefile Tue Sep 20 15:23:05 2016 +0000 +++ b/Makefile Fri Oct 07 12:48:36 2016 +0000 @@ -1,31 +1,19 @@ -.PHONY: clean generate regen serve deploy +.PHONY: clean generate serve deploy + +less := $(shell ffind --literal '.less') +style := $(subst .less,.css,$(less)) -wisps := $(shell ffind --literal '.wisp') -javascripts := $(subst .wisp,.js,$(wisps)) -bundles := $(subst /wisp/,/,$(javascripts)) +static/media/css/%.css: static/media/css/%.less + lessc $< > $@ -generate: $(bundles) - hyde -g -s . +generate: + hugo -t stevelosh clean: - rm -rf ./deploy + rm -rf public serve: - hyde -w -s . -k - -regen: clean generate - -deploy: generate - rsync -avz ./deploy/ sl:/var/www/stevelosh.com + hugo server -t stevelosh -media/js/wisp/%.js: media/js/wisp/%.wisp - cat $< | wisp > $@ - -media/js/terrain1.js: $(javascripts) - browserify media/js/wisp/terrain1.js -do $@ - -media/js/terrain2.js: $(javascripts) - browserify media/js/wisp/terrain2.js -do $@ - -media/js/terrain3.js: $(javascripts) - browserify media/js/wisp/terrain3.js -do $@ +deploy: public + # rsync -avz ./deploy/ sl:/var/www/stevelosh.com diff -r bbf39c61e3fe -r e7bc59b9ebda config.toml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/config.toml Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,13 @@ +baseurl = "http://stevelosh.com/" +languageCode = "en-us" +title = "Steve Losh" +PluralizeListTitles = false +PygmentsCodeFences = true +pygmentsuseclasses = true + +[Params] + DateForm = "January 2, 2006" + DateFormFull = "2006-01-02T15:04:05Z" + +[author] + name = "Steve Losh" diff -r bbf39c61e3fe -r e7bc59b9ebda content/404.html --- a/content/404.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,14 +0,0 @@ -{% extends "_flatpage.html" %} - -{% hyde - title: "Not Found" - exclude: True -%} - - -{% block article %} - -OH GOD HOW DID THIS GET HERE SOMEONE IS NOT GOOD WITH LINK -========================================================== - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/about.html --- a/content/about.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,94 +0,0 @@ -{% extends "skeleton/_base.html" %} -{% hyde - title: "About" -%} - -{% block content %} - - -

{{ page.title|safe|typogrify }}

- -
- {% filter typogrify %} - {% markdown %} -I'm **Steve**. I'm currently living in **Reykjavík, Iceland**. -I do a lot of different things. - -If you want to get in touch with me you can [email -me](mailto:steve@stevelosh.com) or find me on [BitBucket][], [GitHub][], or -[Twitter][]. - -[Twitter]: http://twitter.com/stevelosh/ -[BitBucket]: http://bitbucket.org/sjl/ -[GitHub]: http://github.com/sjl/ - - -Programming ------------ - -I went to [Rochester Institute of Technology](http://rit.edu/) for a degree in -Computer Science. - -I've done a lot of different things in the field so far. Right now the things -that are captivating me are artificial intelligence and clean, simple, -*useful* web applications. - -I find programming beautiful. That might sound strange, but there are aspects -of programming and math that I can't describe any other way. - -You can take a look at my [projects][] to see some of the major stuff I've -worked on, or cut to the chase and look at [my BitBucket account][BitBucket]. - -[projects]: /projects/ - -Photography ----------- - -I like photographing people. Other things as well, but especially people. -I tend to gravitate toward the "art" aspects of photography more than the -"journalism" aspects, but I can appreciate both. - -I shoot digital and film (35mm and medium format) depending on my time limits -and mood. I develop film in my bathroom and print it in a rented darkroom. -When I print digital I use a lab nearby; I don't have the money for a -professional printer just yet. - -Dancing -------- - -I teach blues dancing here in Rochester as part of [Lady Luck Blues][]. -I started blues dancing a few years ago and haven't stopped since. My favorite -aspect of dancing is connection and blues seems to bring that out more than any -other dance. - -I've also been swing dancing for about seven years. My first love is Lindy Hop; I -go to classes, workshops and exchanges as often as my schedule and budget will -allow. - -Lately I've begun getting my feet wet in tango. Whether it'll ever become as -important to me as blues or Lindy remains to be seen. - -[Lady Luck Blues]: http://ladyluckblues.com/ - -Music ------ - -I play electric and upright bass. I've played in a metal band in the past, but -right now my focus is on jazz and blues. - -I DJ at swing and blues dances fairly often. My taste is pretty broad, and -I'll cater to whatever the floor wants, but I tend to personally enjoy -slightly down-tempo, small-group music. Let me know if you need a DJ for a -dance. - -About the Site ------------ - -If you want to know more about my site you should check out the [blog -post][hyderewrite] about how I rewrote it. - -[hyderewrite]: /blog/2010/01/moving-from-django-to-hyde/ - {% endmarkdown %} - {% endfilter %} -
-{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/about/index.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/about/index.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,83 @@ ++++ +date = "2010-09-20T12:34:11Z" +draft = false +title = "About" + ++++ + +I'm **Steve**. I'm currently living in **Reykjavík, Iceland**. +I do a lot of different things. + +If you want to get in touch with me you can [email +me](mailto:steve@stevelosh.com) or find me on [BitBucket][], [GitHub][], or +[Twitter][]. + +[Twitter]: http://twitter.com/stevelosh/ +[BitBucket]: http://bitbucket.org/sjl/ +[GitHub]: http://github.com/sjl/ + +Programming +----------- + +I went to [Rochester Institute of Technology](http://rit.edu/) for a degree in +Computer Science. + +I've done a lot of different things in the field so far. Right now the things +that are captivating me are artificial intelligence and clean, simple, +*useful* web applications. + +I find programming beautiful. That might sound strange, but there are aspects +of programming and math that I can't describe any other way. + +You can take a look at my [projects][] to see some of the major stuff I've +worked on, or cut to the chase and look at [my BitBucket account][BitBucket]. + +[projects]: /projects/ + +Photography +---------- + +I like photographing people. Other things as well, but especially people. +I tend to gravitate toward the "art" aspects of photography more than the +"journalism" aspects, but I can appreciate both. + +I shoot digital and film (35mm and medium format) depending on my time limits +and mood. I develop film in my bathroom and print it in a rented darkroom. +When I print digital I use a lab nearby; I don't have the money for a +professional printer just yet. + +Dancing +------- + +I teach blues dancing here in Rochester as part of [Lady Luck Blues][]. +I started blues dancing a few years ago and haven't stopped since. My favorite +aspect of dancing is connection and blues seems to bring that out more than any +other dance. + +I've also been swing dancing for about seven years. My first love is Lindy Hop; I +go to classes, workshops and exchanges as often as my schedule and budget will +allow. + +Lately I've begun getting my feet wet in tango. Whether it'll ever become as +important to me as blues or Lindy remains to be seen. + +[Lady Luck Blues]: http://ladyluckblues.com/ + +Music +----- + +I play electric and upright bass. I've played in a metal band in the past, but +right now my focus is on jazz and blues. + +I DJ at swing and blues dances fairly often. My taste is pretty broad, and +I'll cater to whatever the floor wants, but I tend to personally enjoy +slightly down-tempo, small-group music. Let me know if you need a DJ for a +dance. + +About the Site +----------- + +If you want to know more about my site you should check out the [blog +post][hyderewrite] about how I rewrote it. + +[hyderewrite]: /blog/2010/01/moving-from-django-to-hyde/ diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2008/02/microsoft-entourage-applescript-frustration.html --- a/content/blog/2008/02/microsoft-entourage-applescript-frustration.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,44 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "Entourage + Applescript = Frustration" - snip: "This is ridiculous." - created: 2008-02-21 15:25:45 -%} - -{% block article %} - -I've been working on a project lately to automate the setup of some rules and -schedules in Microsoft Entourage. This isn't the easiest thing in the world -because Entourage doesn't really support AppleScript for creating rules or -schedules (though it does an admirable job in a lot of other areas). - -We've resorted to GUI scripting to get the job done. This basically means -we're telling our script to click button X, wait a bit, put "abc" in text -field Y, click button Z, etc. This is painful, fragile and slow but it does -what we need in all but one case. - -That case is selecting a folder. The folder list dialog that pops up doesn't -seem to use any of Apple's UI components which makes GUI scripting it nearly -impossible. Only "nearly" though, because there's a way to get around it: the -keyboard. AppleScript can type into a window by sending a series of key codes. -You can select something from the folder list by typing its name, which means -that as long as you know the name (we do) you can select it. Almost. - -There's one more snag: selection by typing will only ever select the first -item in the list. This means that if you want to select "Inbox" and both the -"On My Computer" list and the "youremail@server.com" list are open, you can -only get the first Inbox. Oh, and good luck predicting which one will show up -first. It seemed to be different each time I opened the window. - -Once again the keyboard comes to our rescue. A simple (and excruciatingly -ugly) fix is to make sure that the only account listing that's open is the one -we want. How can we do that? Send an up arrow key code 50 times to move to the -beginning of the list, then left and down codes 50 times. This collapses the -entire tree. Then just type the name of the account to select it, send the -right arrow code to expand the tree, and type the folder name to select it. - -It's painful, but it works. If anyone has a better solution please let me know -so I can rip this monstrosity out of my code and try to forget about it. - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2008/02/microsoft-entourage-applescript-frustration.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2008/02/microsoft-entourage-applescript-frustration.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,43 @@ ++++ +title = "Entourage + Applescript = Frustration" +snip = "This is ridiculous." +date = 2008-02-21T15:25:45Z +draft = false + ++++ + + +## Hello? + +I've been working on a project lately to automate the setup of some rules and +schedules in Microsoft Entourage. This isn't the easiest thing in the world +because Entourage doesn't really support AppleScript for creating rules or +schedules (though it does an admirable job in a lot of other areas). + +We've resorted to GUI scripting to get the job done. This basically means +we're telling our script to click button X, wait a bit, put "abc" in text +field Y, click button Z, etc. This is painful, fragile and slow but it does +what we need in all but one case. + +That case is selecting a folder. The folder list dialog that pops up doesn't +seem to use any of Apple's UI components which makes GUI scripting it nearly +impossible. Only "nearly" though, because there's a way to get around it: the +keyboard. AppleScript can type into a window by sending a series of key codes. +You can select something from the folder list by typing its name, which means +that as long as you know the name (we do) you can select it. Almost. + +There's one more snag: selection by typing will only ever select the first +item in the list. This means that if you want to select "Inbox" and both the +"On My Computer" list and the "youremail@server.com" list are open, you can +only get the first Inbox. Oh, and good luck predicting which one will show up +first. It seemed to be different each time I opened the window. + +Once again the keyboard comes to our rescue. A simple (and excruciatingly +ugly) fix is to make sure that the only account listing that's open is the one +we want. How can we do that? Send an up arrow key code 50 times to move to the +beginning of the list, then left and down codes 50 times. This collapses the +entire tree. Then just type the name of the account to select it, send the +right arrow code to expand the tree, and type the folder name to select it. + +It's painful, but it works. If anyone has a better solution please let me know +so I can rip this monstrosity out of my code and try to forget about it. diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2008/04/shooting-girl-jam.html --- a/content/blog/2008/04/shooting-girl-jam.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,203 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "Shooting Girl Jam" - snip: "I’m finally getting the kind of dancing photos I want." - created: 2008-04-29 18:31:16 -%} - - -{% block article %} - - - GirlJamSaturday-5383 - - -This past weekend (April 25-28) was [Northeast Girl Jam][] in Rochester, New -York. Girl Jam is a swing dancing workshop weekend that focuses on classes for -the followers. It was a huge success; we had a lot of people attend and -everyone seemed to have a great time. The dances had a ton of energy and the -late night parties were wonderful (as always). - -I took a bunch of photographs over the course of the weekend, mostly during -the performances and competitions. Once I posted them I got a bunch of -questions asking me how I did it, so I figured I'd write here about it in more -detail. I used different techniques each day so I'll go through them one by -one. - -Friday ------- - -My goal for the first two nights was to get images with the dancers sharp -enough to recognize but with enough blur to convey the feeling of movement and -energy in the room. I started playing with this technique at the blues parties -in the past and I think I'm really starting to get the hang of it. - - - GirlJamFriday-4873 - - -On Friday I only had one of my flashes with me, so I had to make some -tradeoffs. I bounced the flash from the ceiling to get more even lighting -(directional light from a bare flash is usually too harsh), but since the -ceilings in Tango Cafe are so high it took a lot of power. I wound up shooting -at ISO 1600 and 3200 for most of the night so that my flash could be on a -lower power setting and fire faster. Even at that ISO the noise isn't really -that bad since the photos are exposed well (thanks to the flash). This photo -was shot at ISO 3200 and I don't think the noise distracts from the image much -at all. - - - GirlJamFriday-4943 - - -I was using a wide angle lens (18mm) so that I could get entire bodies into -the frame. One of the things I love about Lindy Hop is that it really uses the -entire body which this photo really shows off. Cropping off huge parts of -people in every single shot makes that much harder to see. I set the aperture -to about f/4 and that gave me enough depth of field to get most things in -focus at 18mm. I set the shutter speed depending on the amount of ambient -light; it varied from 1/30 to 1/4 or so. - - - GirlJamFriday-5007 - - -The trick that really made a difference in taking good photos is that once I -set the exposure I stopped looking at the camera entirely. I didn't review my -shots as I took them and I didn't even look through the viewfinder to compose. -Using a wide lens meant that I could just point the camera in the general -direction of the dancers and still get them. I took this photo at the late -night and the camera was held against my ribs as I did. - -Why did I do that? I can perfectly compose an image but if the dancers aren't -doing something interesting it's going to be a boring photo. I've been dancing -long enough that I'm starting to be able to predict when something cool will -happen in a dance, but that only works if I'm paying complete attention to it. -Messing with the camera distracts me and I can only get the most obvious -moments. Ignoring the camera and watching the dancers means I can pick up more -subtle parts of the dance and capture those (as well as the obvious ones). - -Saturday --------- - -On Saturday I brought along two flashes to the dance and had Sergey hold one -while I held the other (thanks Sergey!). Two flashes means twice as much -light, which means I can shoot with recycle times twice as fast and have more -even light coverage. - - - GirlJamSaturday-5379 - - -Since I was able to shoot twice as many photos I was able to experiment with -getting up close. Using a wide angle lens let me get most of the dancers in -the frame when super close and allowed me to play around with really -interesting perspectives. This photo is one of my favorites from the weekend. -Getting really close to Nina means that the distance between her and Carl is -exaggerated and adds to the sense of tension. Once again, the flashes freeze -the dancers and the ambient light burns in a bit of blur to add some movement. -This photo was also shot at ISO 1600 but the noise is definitely not the main -focus of this picture. As long as you don't underexpose noise is usually not a -problem in these kind of photos. - - - GirlJamSaturday-5349 - - -Not only did I try getting up close, I also tried varying my angle more than I -usually do. I usually brace the camera against my ribs when taking these kinds -of photos for a few reasons: - -* It's a safe height that will get the whole dancer in the frame. -* My ribs are vertical and so aligning the camera with them means that it's - not wildly tilted up or down and I don't accidentally get ceiling- or - floor-only photos. -* It keeps the camera close to my body where it's much less likely to be - whacked by a stray limb. - -Getting lower and higher gives me different perspectives that can have really -nifty results. The problem is that it's much harder to know if the subject is -completely (or even mostly) in the frame when the camera is in an awkward -position. To get a photo like this I probably shot four or five at strange -angles that I deleted. - -One other fun thing to notice: you can see Sergey holding the flash right to -the left of her hips. I probably could have cloned out the flare in Photoshop -but I don't think it really detracts from the image much at all. - -Sunday -------- - - - GirlJamSunday-5647 - - -On Sunday I shot at a few of the workshops since there wasn't a dance. A -workshop has a very different feeling than a dance and so I didn't want to try -the same approach as the other two nights. Instead of using flash and a wide -lens I switched to a fast normal lens (my 50mm f/1.4). The light coming -through the windows was bright enough that I was able to shoot at around 1/60 -at f/2 or f/2.8 and ISO 800 or 1600. - -Once again the noise isn't much of a problem because the photos are exposed -well as this picture shows. The shutter speed is just slow enough to get some -blur at the ends of the limbs but not enough to lose all detail. - - - GirlJamSunday-5512 - - -Since I was using a normal lens instead of a wide angle I had to mostly -abandon the idea of getting big group photos and instead try to capture -individual people as they learned. Using a wide aperture let me isolate the -people from the sea of arms and legs in the backgrounds and gave the images a -soft quality that helps reflect the feeling of the afternoon. - -I really like the soft-yet-directional light that came from the big stained -glass windows combined with the overhead lights. Lately I've been using flash -a lot in my photography and it was fun to get back to using natural light. I'm -going to try to practice with it more in the near future. - -Overall -------- - -Northeast Girl Jam was awesome. I had a great time dancing and photographing -and got to see a lot of old friends (and meet new ones). If you're sad you -missed it there's another event in Rochester next month: [Stompology][]. It's -a weekend of solo jazz and Charleston workshops and awesome swing dances. - -If you'd like to see the rest of the photos I took this weekend I posted them -on [Flickr][]. Feel free to comment there or find me on [Twitter][twsl]; -advice/questions/comments are always appreciated! - -[Flickr]: http://flickr.com/photos/sjl7678/collections/72157604785390431/ -[Stompology]: http://stompology.com/ -[Northeast Girl Jam]: http://jojojackson.com/NEGJ/Home.html -[twsl]: {{links.twsl}} - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2008/04/shooting-girl-jam.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2008/04/shooting-girl-jam.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,199 @@ ++++ +title = "Shooting Girl Jam" +snip = "I'm finally getting the kind of dancing photos I want." +date = 2008-04-29T18:31:16Z +draft = false + ++++ + + + GirlJamSaturday-5383 + + +This past weekend (April 25-28) was [Northeast Girl Jam][] in Rochester, New +York. Girl Jam is a swing dancing workshop weekend that focuses on classes for +the followers. It was a huge success; we had a lot of people attend and +everyone seemed to have a great time. The dances had a ton of energy and the +late night parties were wonderful (as always). + +I took a bunch of photographs over the course of the weekend, mostly during +the performances and competitions. Once I posted them I got a bunch of +questions asking me how I did it, so I figured I'd write here about it in more +detail. I used different techniques each day so I'll go through them one by +one. + +[Northeast Girl Jam]: http://jojojackson.com/NEGJ/Home.html + +Friday +------ + +My goal for the first two nights was to get images with the dancers sharp +enough to recognize but with enough blur to convey the feeling of movement and +energy in the room. I started playing with this technique at the blues parties +in the past and I think I'm really starting to get the hang of it. + + + GirlJamFriday-4873 + + +On Friday I only had one of my flashes with me, so I had to make some +tradeoffs. I bounced the flash from the ceiling to get more even lighting +(directional light from a bare flash is usually too harsh), but since the +ceilings in Tango Cafe are so high it took a lot of power. I wound up shooting +at ISO 1600 and 3200 for most of the night so that my flash could be on a +lower power setting and fire faster. Even at that ISO the noise isn't really +that bad since the photos are exposed well (thanks to the flash). This photo +was shot at ISO 3200 and I don't think the noise distracts from the image much +at all. + + + GirlJamFriday-4943 + + +I was using a wide angle lens (18mm) so that I could get entire bodies into +the frame. One of the things I love about Lindy Hop is that it really uses the +entire body which this photo really shows off. Cropping off huge parts of +people in every single shot makes that much harder to see. I set the aperture +to about f/4 and that gave me enough depth of field to get most things in +focus at 18mm. I set the shutter speed depending on the amount of ambient +light; it varied from 1/30 to 1/4 or so. + + + GirlJamFriday-5007 + + +The trick that really made a difference in taking good photos is that once I +set the exposure I stopped looking at the camera entirely. I didn't review my +shots as I took them and I didn't even look through the viewfinder to compose. +Using a wide lens meant that I could just point the camera in the general +direction of the dancers and still get them. I took this photo at the late +night and the camera was held against my ribs as I did. + +Why did I do that? I can perfectly compose an image but if the dancers aren't +doing something interesting it's going to be a boring photo. I've been dancing +long enough that I'm starting to be able to predict when something cool will +happen in a dance, but that only works if I'm paying complete attention to it. +Messing with the camera distracts me and I can only get the most obvious +moments. Ignoring the camera and watching the dancers means I can pick up more +subtle parts of the dance and capture those (as well as the obvious ones). + +Saturday +-------- + +On Saturday I brought along two flashes to the dance and had Sergey hold one +while I held the other (thanks Sergey!). Two flashes means twice as much +light, which means I can shoot with recycle times twice as fast and have more +even light coverage. + + + GirlJamSaturday-5379 + + +Since I was able to shoot twice as many photos I was able to experiment with +getting up close. Using a wide angle lens let me get most of the dancers in +the frame when super close and allowed me to play around with really +interesting perspectives. This photo is one of my favorites from the weekend. +Getting really close to Nina means that the distance between her and Carl is +exaggerated and adds to the sense of tension. Once again, the flashes freeze +the dancers and the ambient light burns in a bit of blur to add some movement. +This photo was also shot at ISO 1600 but the noise is definitely not the main +focus of this picture. As long as you don't underexpose noise is usually not a +problem in these kind of photos. + + + GirlJamSaturday-5349 + + +Not only did I try getting up close, I also tried varying my angle more than I +usually do. I usually brace the camera against my ribs when taking these kinds +of photos for a few reasons: + +* It's a safe height that will get the whole dancer in the frame. +* My ribs are vertical and so aligning the camera with them means that it's + not wildly tilted up or down and I don't accidentally get ceiling- or + floor-only photos. +* It keeps the camera close to my body where it's much less likely to be + whacked by a stray limb. + +Getting lower and higher gives me different perspectives that can have really +nifty results. The problem is that it's much harder to know if the subject is +completely (or even mostly) in the frame when the camera is in an awkward +position. To get a photo like this I probably shot four or five at strange +angles that I deleted. + +One other fun thing to notice: you can see Sergey holding the flash right to +the left of her hips. I probably could have cloned out the flare in Photoshop +but I don't think it really detracts from the image much at all. + +Sunday +------- + + + GirlJamSunday-5647 + + +On Sunday I shot at a few of the workshops since there wasn't a dance. A +workshop has a very different feeling than a dance and so I didn't want to try +the same approach as the other two nights. Instead of using flash and a wide +lens I switched to a fast normal lens (my 50mm f/1.4). The light coming +through the windows was bright enough that I was able to shoot at around 1/60 +at f/2 or f/2.8 and ISO 800 or 1600. + +Once again the noise isn't much of a problem because the photos are exposed +well as this picture shows. The shutter speed is just slow enough to get some +blur at the ends of the limbs but not enough to lose all detail. + + + GirlJamSunday-5512 + + +Since I was using a normal lens instead of a wide angle I had to mostly +abandon the idea of getting big group photos and instead try to capture +individual people as they learned. Using a wide aperture let me isolate the +people from the sea of arms and legs in the backgrounds and gave the images a +soft quality that helps reflect the feeling of the afternoon. + +I really like the soft-yet-directional light that came from the big stained +glass windows combined with the overhead lights. Lately I've been using flash +a lot in my photography and it was fun to get back to using natural light. I'm +going to try to practice with it more in the near future. + +Overall +------- + +Northeast Girl Jam was awesome. I had a great time dancing and photographing +and got to see a lot of old friends (and meet new ones). If you're sad you +missed it there's another event in Rochester next month: [Stompology][]. It's +a weekend of solo jazz and Charleston workshops and awesome swing dances. + +If you'd like to see the rest of the photos I took this weekend I posted them +on [Flickr][]. Feel free to comment there or find me on [Twitter][twsl]; +advice/questions/comments are always appreciated! + +[Flickr]: http://flickr.com/photos/sjl7678/collections/72157604785390431/ +[Stompology]: http://stompology.com/ +[twsl]: http://twitter.com/stevelosh/ diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2008/08/beauty-in-computer-science-recursion.html --- a/content/blog/2008/08/beauty-in-computer-science-recursion.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,183 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "Beauty in Computer Science" - snip: "Why I love what I do." - created: 2008-08-29 15:30:38 -%} - - -{% block article %} - -When I went to college, I majored in Computer Science. I haven't really -written anything about this part of my life yet, so I figured this might be a -good time to start. - -I decided to major in CS when I was in high school. I learned to program on my -own and enjoyed the challenge of it. I also knew that programming jobs are -fairly common and pay well enough that I wouldn't have to worrying about -paying my rent. I wasn't artistic in high school, and CS seemed like a -logical, sterile field that I felt I could do well in. - -During the years I spent at RIT I changed quite a bit. I think the most -important, or at least the most obvious, change has been my becoming more -artistic. Between dancing, playing bass and photography I've developed a -creative side that I never thought I could. Has this led me away from Computer -Science at all? - -No. While I became a programmer for practical reasons, I stayed one for -entirely different reasons. Computer Science (and math in general) is -beautiful. It took me years to slowly realize this, but now that I have I see -that it's more beautiful than any dance, photograph or music I've ever -encountered. This post is the first in a series of posts that I hope can -communicate why I find CS beautiful, or at least point in the right direction. - -A Quick Primer on "Functions" for Non-Programmers ---------- - -I know that most (if not all) of the people that read my website don't -program. I'm going to try to avoid using extremely technical, computer sciency -terms in these posts, but there are at least a few basic things that I need to -explain. The first is what a function is. - -Think of a function as a set of instructions. A recipe is a decent example. -Imagine you have a piece of paper with the following written on it: - -1. Heat a frying pan. -2. Crack two eggs into a bowl. -3. Mix the eggs. -4. Pour the mixture into the hot frying pan. -5. Stir the mixture until it is solid. -6. Take the mixture out of the pan. - -That's the simplest example of a function: a list of instructions that you -follow in the order you get them. The next idea is a "parameter." That recipe -only told us how to make food for one person, wouldn't it be nice to know how -to make enough for two or more? Imagine the paper now says this: - -1. Heat a frying pan. -2. Crack NUMBER_OF_PEOPLE * 2 eggs into a bowl. -3. Mix the eggs. -4. Pour the mixture into the hot frying pan. -5. Stir the mixture until it is solid. -6. Take the mixture out of the pan. - -Now we still have a single piece of paper, but by just substituting in the -number of people we're cooking for we have instructions for making any amount -of food. If we have three people, NUMBER_OF_PEOPLE * 2 eggs becomes 3 * 2 -eggs, or 6 eggs. - -The last important concept is "calling." Once you have one function, you can -use it in other functions, like so: - -1. Put a plate on the table. -2. Refer to the other piece of paper and do what it says, for 1 person. -3. Put the result of that on the plate. -4. Eat it. - -See how we did a few things, then referred to the other piece of paper instead -of repeating ourselves? This lets you make small tasks and put them together -to form bigger ones. That's enough of a primer for now; if something's not -clear please find me on [Twitter][twsl] and let me know. - -[twsl]: {{links.twsl}} - -What's Recursion? ---------------- - -Recursion is a term that means, basically, a function calls (or refers to) -itself. This concept can be hard to grasp, but once you do it slowly turns -into something breathtakingly beautiful. - -Here's a simple example: imagine you're 10 meters away from a doorway and you -want to walk out of it. A function that could help you do this might be: "Walk -11 meters." That's pretty simple. - -What if we want to be able to walk out of a doorway that's any number of -meters ahead of us? We could say: "Walk DISTANCE + 1 meters." This works, but -requires that you know how many meters away the door is before you even start -walking. What if we don't? - -How about we just do a little at a time and see how it goes? "Walk 2 meters. -If you're out of the doorway, stop. Otherwise, repeat this function." If the -doorway is 3 meters in front of us, we'll walk 2, check if we're out (we're -not), and repeat. We'll walk 2, check if we're out (we are), and stop. This -function is recursive; it refers back to itself. - -Why is it beautiful? --------------- - -The beauty of recursion, for me, is that you can build infinitely large or -complex tasks or structures using one or two tiny, simple parts. I'll give an -example of this because I think it's the easiest way for me to show it. - -Imagine that you only know three things: how to add 1 to a number, how to -subtract 1 from a number, and what 0 means. Now imagine that you want to be -able to add any two (positive) numbers (integers) together. How could you do -this? Here's a function: - - :::text - function add(x, y): - if y = 0: - return x - otherwise: - return add( x+1, y-1 ) - -Don't let the new notation scare you; it's the same kind of function as -always. In English, it says "This function is named 'add' and needs two -parameters (x and y). Step one is to see if y is 0. If it is, the result is x. -If it's not, then the result is whatever you get from performing this function -with the following parameters: x+1, y-1." - -It might not be immediately obvious that this will actually perform the -addition we're used to, so let's try it. First, let's try adding 1 + 0. In -that case, we check if y is 0. It is, so the result is x, or 1. So far, so -good. - -Let's try 2 + 1. Is y equal to 0? No, so we need to call add with some new -parameters. 2+1 is 3, 1-0 is 0, so the result is now whatever we get from -add(3, 0). We start following the instructions of add. Does y equal 0? Yes, so -the result is 3, which is what we expect. - -One more for good measure: 9 + 2. First we check if y is 0. It's not, so the -result is then add(9+1, 2-1), or add(10,1). Start over. Is y zero? Nope, so -the result is now add(11, 0). Start over. Is y zero? Yes, so the result is x, -or 11. - -So What? --------- - -This will work for any x and any y (as long as they're both positive, negative -numbers make it just a tiny bit more complicated). This might not seem -impressive, but think about what we've done. We've created something that can -add any two numbers together. There are an infinite amount of numbers. This -tiny function that we've made from the simplest of parts can generate more -results than there are people on this planet. Or atoms in the universe. To me, -this is amazing. Not "magic trick" amazing or "miracle" amazing but "I can -understand and create something that can describe more knowledge than the -whole of humanity couldn't hope to describe in a thousand lifetimes" amazing. - -Computer Science (and math in general) is full of this kind of beauty. I've -tried to find parallels in my other interests; the closest I've found is the -photograph [Pale Blue Dot][]. It's a photo taken by the Voyager 1 space probe -showing an immense amount of space with the tiniest blue speck in the middle, -which is Earth. - -When you view the photograph, it's a mostly black field with a little fleck of -blue pigment. Not terribly complicated or interesting, until you realize that -that blue fleck is a representation of the planet that billions of people have -lived and died on. A smidgen of blue ink, in the right context, represents -every place a human has ever called "home." - -Our addition function might not seems nearly as nifty as this, but it actually -represents more than even this photo ever can. There are a finite number of -people on Earth, but the addition function can add more numbers that that. -Even if you count every second in the life of every person that has ever lived -on our planet, the addition function can create more numbers than that. - -This awes me more than any myth, photograph, story, song, legend or dance ever -has. It's the reason I stayed a Computer Scientist. - -[Pale Blue Dot]: http://en.wikipedia.org/wiki/Pale_Blue_Dot - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2008/08/beauty-in-computer-science-recursion.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2008/08/beauty-in-computer-science-recursion.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,180 @@ ++++ +title = "Beauty in Computer Science" +snip = "Why I love what I do." +date = 2008-08-29T15:30:38Z +draft = false + ++++ + +When I went to college, I majored in Computer Science. I haven't really +written anything about this part of my life yet, so I figured this might be a +good time to start. + +I decided to major in CS when I was in high school. I learned to program on my +own and enjoyed the challenge of it. I also knew that programming jobs are +fairly common and pay well enough that I wouldn't have to worrying about +paying my rent. I wasn't artistic in high school, and CS seemed like a +logical, sterile field that I felt I could do well in. + +During the years I spent at RIT I changed quite a bit. I think the most +important, or at least the most obvious, change has been my becoming more +artistic. Between dancing, playing bass and photography I've developed a +creative side that I never thought I could. Has this led me away from Computer +Science at all? + +No. While I became a programmer for practical reasons, I stayed one for +entirely different reasons. Computer Science (and math in general) is +beautiful. It took me years to slowly realize this, but now that I have I see +that it's more beautiful than any dance, photograph or music I've ever +encountered. This post is the first in a series of posts that I hope can +communicate why I find CS beautiful, or at least point in the right direction. + +A Quick Primer on "Functions" for Non-Programmers +--------- + +I know that most (if not all) of the people that read my website don't +program. I'm going to try to avoid using extremely technical, computer sciency +terms in these posts, but there are at least a few basic things that I need to +explain. The first is what a function is. + +Think of a function as a set of instructions. A recipe is a decent example. +Imagine you have a piece of paper with the following written on it: + +1. Heat a frying pan. +2. Crack two eggs into a bowl. +3. Mix the eggs. +4. Pour the mixture into the hot frying pan. +5. Stir the mixture until it is solid. +6. Take the mixture out of the pan. + +That's the simplest example of a function: a list of instructions that you +follow in the order you get them. The next idea is a "parameter." That recipe +only told us how to make food for one person, wouldn't it be nice to know how +to make enough for two or more? Imagine the paper now says this: + +1. Heat a frying pan. +2. Crack NUMBER_OF_PEOPLE * 2 eggs into a bowl. +3. Mix the eggs. +4. Pour the mixture into the hot frying pan. +5. Stir the mixture until it is solid. +6. Take the mixture out of the pan. + +Now we still have a single piece of paper, but by just substituting in the +number of people we're cooking for we have instructions for making any amount +of food. If we have three people, NUMBER_OF_PEOPLE * 2 eggs becomes 3 * 2 +eggs, or 6 eggs. + +The last important concept is "calling." Once you have one function, you can +use it in other functions, like so: + +1. Put a plate on the table. +2. Refer to the other piece of paper and do what it says, for 1 person. +3. Put the result of that on the plate. +4. Eat it. + +See how we did a few things, then referred to the other piece of paper instead +of repeating ourselves? This lets you make small tasks and put them together +to form bigger ones. That's enough of a primer for now; if something's not +clear please find me on [Twitter][twsl] and let me know. + +[twsl]: {{links.twsl}} + +What's Recursion? +--------------- + +Recursion is a term that means, basically, a function calls (or refers to) +itself. This concept can be hard to grasp, but once you do it slowly turns +into something breathtakingly beautiful. + +Here's a simple example: imagine you're 10 meters away from a doorway and you +want to walk out of it. A function that could help you do this might be: "Walk +11 meters." That's pretty simple. + +What if we want to be able to walk out of a doorway that's any number of +meters ahead of us? We could say: "Walk DISTANCE + 1 meters." This works, but +requires that you know how many meters away the door is before you even start +walking. What if we don't? + +How about we just do a little at a time and see how it goes? "Walk 2 meters. +If you're out of the doorway, stop. Otherwise, repeat this function." If the +doorway is 3 meters in front of us, we'll walk 2, check if we're out (we're +not), and repeat. We'll walk 2, check if we're out (we are), and stop. This +function is recursive; it refers back to itself. + +Why is it beautiful? +-------------- + +The beauty of recursion, for me, is that you can build infinitely large or +complex tasks or structures using one or two tiny, simple parts. I'll give an +example of this because I think it's the easiest way for me to show it. + +Imagine that you only know three things: how to add 1 to a number, how to +subtract 1 from a number, and what 0 means. Now imagine that you want to be +able to add any two (positive) numbers (integers) together. How could you do +this? Here's a function: + +```text +function add(x, y): + if y = 0: + return x + otherwise: + return add( x+1, y-1 ) +``` + +Don't let the new notation scare you; it's the same kind of function as +always. In English, it says "This function is named 'add' and needs two +parameters (x and y). Step one is to see if y is 0. If it is, the result is x. +If it's not, then the result is whatever you get from performing this function +with the following parameters: x+1, y-1." + +It might not be immediately obvious that this will actually perform the +addition we're used to, so let's try it. First, let's try adding 1 + 0. In +that case, we check if y is 0. It is, so the result is x, or 1. So far, so +good. + +Let's try 2 + 1. Is y equal to 0? No, so we need to call add with some new +parameters. 2+1 is 3, 1-0 is 0, so the result is now whatever we get from +add(3, 0). We start following the instructions of add. Does y equal 0? Yes, so +the result is 3, which is what we expect. + +One more for good measure: 9 + 2. First we check if y is 0. It's not, so the +result is then add(9+1, 2-1), or add(10,1). Start over. Is y zero? Nope, so +the result is now add(11, 0). Start over. Is y zero? Yes, so the result is x, +or 11. + +So What? +-------- + +This will work for any x and any y (as long as they're both positive, negative +numbers make it just a tiny bit more complicated). This might not seem +impressive, but think about what we've done. We've created something that can +add any two numbers together. There are an infinite amount of numbers. This +tiny function that we've made from the simplest of parts can generate more +results than there are people on this planet. Or atoms in the universe. To me, +this is amazing. Not "magic trick" amazing or "miracle" amazing but "I can +understand and create something that can describe more knowledge than the +whole of humanity couldn't hope to describe in a thousand lifetimes" amazing. + +Computer Science (and math in general) is full of this kind of beauty. I've +tried to find parallels in my other interests; the closest I've found is the +photograph [Pale Blue Dot][]. It's a photo taken by the Voyager 1 space probe +showing an immense amount of space with the tiniest blue speck in the middle, +which is Earth. + +When you view the photograph, it's a mostly black field with a little fleck of +blue pigment. Not terribly complicated or interesting, until you realize that +that blue fleck is a representation of the planet that billions of people have +lived and died on. A smidgen of blue ink, in the right context, represents +every place a human has ever called "home." + +Our addition function might not seems nearly as nifty as this, but it actually +represents more than even this photo ever can. There are a finite number of +people on Earth, but the addition function can add more numbers that that. +Even if you count every second in the life of every person that has ever lived +on our planet, the addition function can create more numbers than that. + +This awes me more than any myth, photograph, story, song, legend or dance ever +has. It's the reason I stayed a Computer Scientist. + +[Pale Blue Dot]: http://en.wikipedia.org/wiki/Pale_Blue_Dot + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2008/08/negative-space-dancing.html --- a/content/blog/2008/08/negative-space-dancing.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,103 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "Negative Space in Dancing" - snip: "It’s not just for artsy kids." - created: 2008-08-31 15:33:57 -%} - - -{% block article %} - -Last night a few of us from Rochester drove to Buffalo for a swing event. -There were three classes during the afternoon that were pretty fun. There was -a dance in the evening which had a great turnout and plenty of energy. As nice -as all that was, the real highlight of the day/night was the blues party after -the dance. Matt and I ended up DJ'ing it and it was absolutely awesome. I got -to meet and dance with a whole bunch of new people that I hope to dance with -again soon. - -One concept I try to use in my blues (and tango) dancing is that of negative -space. I've found that a lot of blues dancers don't really seem to quite get -that idea. I'm not saying that it's the "right" way to blues dance but it's -something that's changed how I dance and I wanted to write something about it -to share it with anyone interested. - -What is negative space? -------------------- - -Negative space is a term used a lot in painting, photography, graphic design, -etc. It basically means that for a given photo (or painting, whatever) the -whole frame isn't filled with "things." Two examples of this taken from my -flickr favorites are [this photo][1] and [this one][2]. In both of these -images the subject takes up only a small portion of the frame. The expanses of -even texture and tone add certain qualities to the photos that can't be -achieved otherwise. - -A photographer that takes this to an extreme is Hiroshi Sugimoto, in his -[Seascape][] series. He uses long exposures and darkroom techniques to produce -images of nothing but negative space. The sea blends into a seamless texture -and the sky becomes a single tone. Even though there isn't a physical object -between the two, the division between them forms a subject of its own. - -How does it relate to dancing? ------------------------- - -Most of the blues dancers I know come from a Lindy Hop background. Lindy Hop -is a dance of movement and momentum; that's part of what makes it so much fun. -When blues dancing I think a lot of Lindy Hoppers never catch on to this idea: -*you don't need to be moving all of the time*. It's alright to slow down and -actually stop for one, two, three, four measures. It gives the dance a chance -to breathe and lets you focus on your connection with your partner without -movement getting in the way. - -"Won't that be boring?" No. Absolutely not. Stopping can provide things that -you can't really get while moving. It can add tension, release it, and -completely change the mood of a dance. It really does have a lot in common -with negative space in art, and it adds a lot to a dance. - -Negative space does not mean "empty space," although it's often described like -that. In the first photo I linked to the wall is indeed blank, but has color -and texture that can give you plenty to look at. In the second, the expanses -of white are just that: white. They are not empty; they have a distinct tone: -white. If they were black, or even a light grey the photo would be entirely -different. - -In dancing, movement is important but not the only thing we think about. -Posture and body positioning is one example. "Feeling" or "mood" is another, -more elusive, one. Just because the movement is gone doesn't mean all these -other things go away automatically; they're still there until you let them go. - -Taking it to the extreme like Sugimoto's Seascapes is also emphatically *not* -boring. In some of the best blues dances I've ever had there were parts where -we didn't really move for several measures at a time. When this happens, both -of you create your own negative space through your posture, etc. Neither of -these spaces are empty. - -Where these spaces meet is your connection. Like the horizons in Sugimoto's -photographs it becomes the main subject. By eliminating motion you're free to -focus your attention on other aspects of the connection. Sometimes these -aren't obvious at first. One example Mihai likes to talk about is breathing. -Another is understanding where your partner's weight is. - -By giving yourself time to really feel the connection with your partner you -have time to examine it much more closely. You can introduce tiny movements -(like the waves that fleck the surface of the water in Sugimito's seas) that -interrupt the absolute flatness of the spaces and let you explore the -interactions between them. Not only does this let you learn about how your -partner is connected to you in a more in-depth way, it's very, very fun. - -I don't have a particularly amazing ending for this post. The main points I -wanted to get across are these: Negative space in dancing is removing most or -all of the movement for a longer-than-normal about of time. Negative space is -not empty space unless you ignore all the other aspects of the dance like -posture, mood and connection. Negative space can add things to a dance that -you can't get otherwise. - -Try it. - -[1]: http://flickr.com/photos/theoperamafia/2790037114/ -[2]: http://flickr.com/photos/jodiseva/2385347094/ -[Seascape]: http://www.sugimotohiroshi.com/seascape.html - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2008/08/negative-space-dancing.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2008/08/negative-space-dancing.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,99 @@ ++++ +title = "Negative Space in Dancing" +snip = "It’s not just for artsy kids." +date = 2008-08-31T15:33:57Z +draft = false + ++++ + +Last night a few of us from Rochester drove to Buffalo for a swing event. +There were three classes during the afternoon that were pretty fun. There was +a dance in the evening which had a great turnout and plenty of energy. As nice +as all that was, the real highlight of the day/night was the blues party after +the dance. Matt and I ended up DJ'ing it and it was absolutely awesome. I got +to meet and dance with a whole bunch of new people that I hope to dance with +again soon. + +One concept I try to use in my blues (and tango) dancing is that of negative +space. I've found that a lot of blues dancers don't really seem to quite get +that idea. I'm not saying that it's the "right" way to blues dance but it's +something that's changed how I dance and I wanted to write something about it +to share it with anyone interested. + +What is negative space? +------------------- + +Negative space is a term used a lot in painting, photography, graphic design, +etc. It basically means that for a given photo (or painting, whatever) the +whole frame isn't filled with "things." Two examples of this taken from my +flickr favorites are [this photo][1] and [this one][2]. In both of these +images the subject takes up only a small portion of the frame. The expanses of +even texture and tone add certain qualities to the photos that can't be +achieved otherwise. + +A photographer that takes this to an extreme is Hiroshi Sugimoto, in his +[Seascape][] series. He uses long exposures and darkroom techniques to produce +images of nothing but negative space. The sea blends into a seamless texture +and the sky becomes a single tone. Even though there isn't a physical object +between the two, the division between them forms a subject of its own. + +How does it relate to dancing? +------------------------ + +Most of the blues dancers I know come from a Lindy Hop background. Lindy Hop +is a dance of movement and momentum; that's part of what makes it so much fun. +When blues dancing I think a lot of Lindy Hoppers never catch on to this idea: +*you don't need to be moving all of the time*. It's alright to slow down and +actually stop for one, two, three, four measures. It gives the dance a chance +to breathe and lets you focus on your connection with your partner without +movement getting in the way. + +"Won't that be boring?" No. Absolutely not. Stopping can provide things that +you can't really get while moving. It can add tension, release it, and +completely change the mood of a dance. It really does have a lot in common +with negative space in art, and it adds a lot to a dance. + +Negative space does not mean "empty space," although it's often described like +that. In the first photo I linked to the wall is indeed blank, but has color +and texture that can give you plenty to look at. In the second, the expanses +of white are just that: white. They are not empty; they have a distinct tone: +white. If they were black, or even a light grey the photo would be entirely +different. + +In dancing, movement is important but not the only thing we think about. +Posture and body positioning is one example. "Feeling" or "mood" is another, +more elusive, one. Just because the movement is gone doesn't mean all these +other things go away automatically; they're still there until you let them go. + +Taking it to the extreme like Sugimoto's Seascapes is also emphatically *not* +boring. In some of the best blues dances I've ever had there were parts where +we didn't really move for several measures at a time. When this happens, both +of you create your own negative space through your posture, etc. Neither of +these spaces are empty. + +Where these spaces meet is your connection. Like the horizons in Sugimoto's +photographs it becomes the main subject. By eliminating motion you're free to +focus your attention on other aspects of the connection. Sometimes these +aren't obvious at first. One example Mihai likes to talk about is breathing. +Another is understanding where your partner's weight is. + +By giving yourself time to really feel the connection with your partner you +have time to examine it much more closely. You can introduce tiny movements +(like the waves that fleck the surface of the water in Sugimito's seas) that +interrupt the absolute flatness of the spaces and let you explore the +interactions between them. Not only does this let you learn about how your +partner is connected to you in a more in-depth way, it's very, very fun. + +I don't have a particularly amazing ending for this post. The main points I +wanted to get across are these: Negative space in dancing is removing most or +all of the movement for a longer-than-normal about of time. Negative space is +not empty space unless you ignore all the other aspects of the dance like +posture, mood and connection. Negative space can add things to a dance that +you can't get otherwise. + +Try it. + +[1]: http://flickr.com/photos/theoperamafia/2790037114/ +[2]: http://flickr.com/photos/jodiseva/2385347094/ +[Seascape]: http://www.sugimotohiroshi.com/seascape.html + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2008/08/on-leading.html --- a/content/blog/2008/08/on-leading.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,90 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "On Leading" - snip: "Some of my thoughts on leading after five years of doing it." - created: 2008-08-01 15:28:33 -%} - - -{% block article %} - -For those of you that don't know, one of the things I do with my free time is -dancing. I've been swing dancing (Lindy Hop) for about five years, blues -dancing for a year or so, and recently started learning tango. All of these -dances are improvised partner dances and so rely heavily on leading and -following. People do make routines but at least 95% of it is unrehearsed -social dancing with partners you might have never met. - -As a male I'm usually in the role of leader, though I do try to follow when I -get the chance. I've learned a lot over the years so I'm going to write a few -posts about leading, and this is the first. I'm going to use the traditional -pronouns to make things easier to read, but everything applies to both genders -in both roles. - -Beginning ------ - -When a guy is first taught how to swing dance (or blues, or tango; everything -I'm saying applies to all three) he's usually taught that his main job is to -lead. This sounds obvious, but it's a lot for a beginner to take in. He has to -learn the footwork and ingrain it into his memory so it becomes automatic, -which takes some time. The next step is learning individual moves: not only -how to do them himself but also how to lead a follower to do them at the same -time. It takes coordination and most of all practice. - -Leading at this point involves clearly showing the follower where she should -go and what she should do. "Placing the follower's weight" is a concept that's -a bit tricky but very useful. If a leader isn't clear in his leading the -follower won't be able to follow him unless she "cheats" and just does what -she knows he wants her to do (because she's danced with him before and so -knows what he's trying to do). This falls apart when the leader dances with a -new partner. Without leading and following swing dancing just doesn't work, so -leading clearly is the main role of a beginner guy. - -Moving On ------ - -Let me take a second to explain something I see happen very often with leaders -that take classes and progress nicely in their skill. Once the leader gets the -basics down pat and starts learning more moves, there seems to be a tendency -to learn things that let him show off. The followers get to really shine in -Lindy Hop quite a bit, so it's only natural for the guys to want to measure up -and look cool themselves. Unfortunately I think this gets in the way of my -next idea. - -I think once a leader reaches a point where he's comfortable with the -structure of the dance and has a repertoire of moves and vocabulary of -movements, his role changes. His job is no longer "lead." His role becomes -*"lead the follower you are dancing with right now."* - -Every follower is different. Every single one has a different level of -experience, a different style, and a different personality (as it relates to -dancing). If the leader simply leads every dance the same way, those dances -are not as good as they could be. An "advanced" leader leading a beginner -follower in a lot of complicated movements she's not capable of following yet -turns into a complete mess. He goes away from the dance feeling bored or -frustrated (or worse, arrogant) and she goes away feeling confused, -discouraged or angry. This is not a good thing. - -Paying attention to the follow's level is critical. I'm not saying "only do -moves that the follower has learned and can easily follow." Pushing the -follower slightly beyond her comfortable, "automatic" level is wonderful and -helps her immensely; but going totally over her head and confusing the hell -out of her just so he can show off (to her or others) is obnoxious. This also -works in reverse: followers, please challenge your leaders but be mindful of -their skill. - -Experience isn't the only difference between followers. Each follower has her -own style that won't always fit perfectly with the leader's personal style. -Adjusting his style to mesh better with hers makes the connection between -partners so much better, which makes the dance that much more fun. This also -works both ways. Followers are generally better at "listening" to their -partner because it's their main job; if a lead makes an effort to really -listen to the follow and change his leading to incorporate her ideas, -personality, style and level it makes an enormous difference. - -The point I'm trying to make is that "leading" a follower is not just leading. -It's paying attention to the follower and leading *her*. - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2008/08/on-leading.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2008/08/on-leading.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,86 @@ ++++ +title = "On Leading" +snip = "Some of my thoughts on leading after five years of doing it." +date = 2008-08-01T15:28:33Z +draft = false + ++++ + +For those of you that don't know, one of the things I do with my free time is +dancing. I've been swing dancing (Lindy Hop) for about five years, blues +dancing for a year or so, and recently started learning tango. All of these +dances are improvised partner dances and so rely heavily on leading and +following. People do make routines but at least 95% of it is unrehearsed +social dancing with partners you might have never met. + +As a male I'm usually in the role of leader, though I do try to follow when I +get the chance. I've learned a lot over the years so I'm going to write a few +posts about leading, and this is the first. I'm going to use the traditional +pronouns to make things easier to read, but everything applies to both genders +in both roles. + +Beginning +----- + +When a guy is first taught how to swing dance (or blues, or tango; everything +I'm saying applies to all three) he's usually taught that his main job is to +lead. This sounds obvious, but it's a lot for a beginner to take in. He has to +learn the footwork and ingrain it into his memory so it becomes automatic, +which takes some time. The next step is learning individual moves: not only +how to do them himself but also how to lead a follower to do them at the same +time. It takes coordination and most of all practice. + +Leading at this point involves clearly showing the follower where she should +go and what she should do. "Placing the follower's weight" is a concept that's +a bit tricky but very useful. If a leader isn't clear in his leading the +follower won't be able to follow him unless she "cheats" and just does what +she knows he wants her to do (because she's danced with him before and so +knows what he's trying to do). This falls apart when the leader dances with a +new partner. Without leading and following swing dancing just doesn't work, so +leading clearly is the main role of a beginner guy. + +Moving On +----- + +Let me take a second to explain something I see happen very often with leaders +that take classes and progress nicely in their skill. Once the leader gets the +basics down pat and starts learning more moves, there seems to be a tendency +to learn things that let him show off. The followers get to really shine in +Lindy Hop quite a bit, so it's only natural for the guys to want to measure up +and look cool themselves. Unfortunately I think this gets in the way of my +next idea. + +I think once a leader reaches a point where he's comfortable with the +structure of the dance and has a repertoire of moves and vocabulary of +movements, his role changes. His job is no longer "lead." His role becomes +*"lead the follower you are dancing with right now."* + +Every follower is different. Every single one has a different level of +experience, a different style, and a different personality (as it relates to +dancing). If the leader simply leads every dance the same way, those dances +are not as good as they could be. An "advanced" leader leading a beginner +follower in a lot of complicated movements she's not capable of following yet +turns into a complete mess. He goes away from the dance feeling bored or +frustrated (or worse, arrogant) and she goes away feeling confused, +discouraged or angry. This is not a good thing. + +Paying attention to the follow's level is critical. I'm not saying "only do +moves that the follower has learned and can easily follow." Pushing the +follower slightly beyond her comfortable, "automatic" level is wonderful and +helps her immensely; but going totally over her head and confusing the hell +out of her just so he can show off (to her or others) is obnoxious. This also +works in reverse: followers, please challenge your leaders but be mindful of +their skill. + +Experience isn't the only difference between followers. Each follower has her +own style that won't always fit perfectly with the leader's personal style. +Adjusting his style to mesh better with hers makes the connection between +partners so much better, which makes the dance that much more fun. This also +works both ways. Followers are generally better at "listening" to their +partner because it's their main job; if a lead makes an effort to really +listen to the follow and change his leading to incorporate her ideas, +personality, style and level it makes an enormous difference. + +The point I'm trying to make is that "leading" a follower is not just leading. +It's paying attention to the follower and leading *her*. + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2009/01/deploying-site-fabric-and-mercurial.html --- a/content/blog/2009/01/deploying-site-fabric-and-mercurial.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,319 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "Deploying with Fabric & Mercurial" - snip: "Trimming typing." - created: 2009-01-15 20:51:09 -%} - -{% block article %} - -Earlier tonight I added support for the [Mint][] [Bird Feeder][] plugin to my -site's [RSS feeds][]. Bird Feeder isn't designed to work with [Django][] so I -had to change a few things to get it up and running. I mostly followed the -[instructions on -Hicks-Wright.net](http://hicks-wright.net/blog/minty-django-feeds/) with a few -tweaks. - -While I was trying to get things running smoothly I had to redeploy the site a -bunch of times so I could test the changes I made. If I were simply FTP'ing -the files over each time I wanted to redeploy it would have been a huge pain. -Fortunately, I have another way to do it that cuts down on my typing. - -[Mint]: http://haveamint.com -[Bird Feeder]: http://haveamint.com/peppermill/pepper/11/bird_feeder/ -[RSS feeds]: /rss/ -[Django]: {{links.django}} - -[TOC] - -My Basic Deployment Steps -------------------------- - -The code for my site is stored in a Mercurial repository. There are actually -three copies of the repository that I use: - -* One is on my local machine, which I commit to as I make changes. -* One is on [BitBucket][], which I use to share the code with the public. -* One is on my host's webserver, and is what actually gets served as the website. - -When I'm ready to deploy a new version of the site, I push the changes I've -made on my local repository to the one on BitBucket, then pull the changes -from BitBucket down to the server. Using push/pull means I don't have to worry -about transferring the files myself. Using BitBucket in the middle (instead of -going right from my local machine to the server) means I don't have to worry -about serving either repository myself and dealing with port forwarding or -security issues. - -[BitBucket]: http://bitbucket.org - -Putting BitBucket in the Middle -------------------------------- - -Using BitBucket as an intermediate repository is actually fairly simple. Here -are the basic steps I use to get a project up and running like this. - -First, create a new repository on BitBucket. Make sure the name is what you -want. Feel free to make it private if you just want it for this, or public if -you want to [go open source][]. - -[go open source]: /blog/entry/2009/1/13/going-open-source/ - -### Set Up Your Local Machine - -Clone this new, empty repository to your local machine. If you already have -code written you'll need to copy it into this folder after cloning. I don't -know if there's a way to push a brand-new repository to BitBucket; if you know -how please tell me. - -On your local machine, edit the `.hg/hgrc` file in the repository and change -the default path to: - - :::ini - default = ssh://hg@bitbucket.org/username/repositoryname/ - -That will let you push/pull to and from BitBucket over SSH. Doing it that way -means you can use public/private key authentication and avoid typing your -password every single time. A fairly comprehensive guide to that can be found -[here](http://www.securityfocus.com/infocus/1810). You can ignore the -server-side configuration; you just need to add your public key on BitBucket's -account settings page and you should be set. - -**UPDATE:** I didn't realize it before, but BitBucket has a [guide to using SSH with BitBucket](http://bitbucket.org/help/using-ssh/). It's definitely worth looking at. - -Now you should be able to use `hg push` and `hg pull` on your local machine to -push and pull changes to and from the BitBucket repository. The next step is -getting it set up on the server side. - -### Set Up Your Server - -On your server, use `hg clone` to clone the BitBucket repository to wherever -you want to serve it from. Edit the `.hg/hgrc` file in that one and change the -default path to the same value as before: - - :::ini - default = ssh://hg@bitbucket.org/username/repositoryname/ - -Once again, set up public/private key authentication; this time between the -server and BitBucket. You can either copy your public and private keys from -your local machine to the server (if you trust/own it) or you can create a new -pair and add its public key to your BitBucket account as well. - -While you're at it, set up public/private key authentication to go from your -local machine to your server too. It'll pay off in the long run. - -Now that you've got both sides working, you can develop and deploy like so: - -* Make changes on your local machine, committing as you go. -* Push the changes to BitBucket. -* SSH into your server and pull the changes down. -* Restart the web server process if necessary. - -Not too bad! Instead of manually managing file transfers you can let Mercurial -do it for you. It'll only pull down the files that have changed, and will -always put them in exactly the right spot. That's pretty convenient, but we -can do better. - -Weaving it All Together with Fabric ------------------------------------ - -Being able to push and pull is all well and good, but that's still a lot of -typing. You need to enter a command to push your changes, a command to SSH to -the server, a command to change to the deploy directory, a command to pull the -changes, and a command to restart the server process. That's five commands, -which is four commands too many for me. - -To automate the process, I use the wonderful [Fabric][] tool. If you haven't -seen it before you should take a look. To follow along with the rest of the -section you should read the examples on the site and install Fabric on your -local machine. - -### My Current Setup - -Here's the fabfile I use for deploying my site to my host ([WebFaction][]). -It's pretty specific to my needs but I'm sure it will give you an idea of -where to start. - -[Fabric]: {{links.fabric}} -[WebFaction]: {{links.webfaction}} - - :::python - def prod(): - """Set the target to production.""" - set(fab_hosts=['sjl.webfactional.com']) - set(fab_key_filename='/Users/sjl/.ssh/stevelosh') - set(remote_app_dir='~/webapps/stevelosh/stevelosh') - set(remote_apache_dir='~/webapps/stevelosh/apache2') - - def deploy(): - """Deploy the site.""" - require('fab_hosts', provided_by = [prod,]) - local("hg push") - run("cd $(remote_app_dir); hg pull; hg update") - run("cd $(remote_app_dir); python2.5 manage.py syncdb") - run("$(remote_apache_dir)/bin/stop; sleep 1; $(remote_apache_dir)/bin/start") - - def debugon(): - """Turn debug mode on for the production server.""" - require('fab_hosts', provided_by = [prod,]) - run("cd $(remote_app_dir); sed -i -e 's/DEBUG = .*/DEBUG = True/' deploy.py") - run("$(remote_apache_dir)/bin/stop; sleep 1; $(remote_apache_dir)/bin/start") - - def debugoff(): - """Turn debug mode off for the production server.""" - require('fab_hosts', provided_by = [prod,]) - run("cd $(remote_app_dir); sed -i -e 's/DEBUG = .*/DEBUG = False/' deploy.py") - run("$(remote_apache_dir)/bin/stop; sleep 1; $(remote_apache_dir)/bin/start") - -When I'm finished committing to my local repository and I want to deploy the -site, I just use the command `fab prod deploy` on my local machine. Fabric -pushes my local repository to BitBucket, logs into the server (with my public -key—no typing in passwords), pulls down the new changes from BitBucket -and restarts the server process. - -I also set up a couple of debug commands so I can type `fab prod debugon` and -`fab prod debugoff` to change the `settings.DEBUG` option of my Django app. -Sometimes it's useful to turn on debug to find out exactly why a page is -breaking on the server. - -### Extending It - -The reason I split off the `prod` command is so I can set up a separate test -app (on the server) in the future and reuse the `deploy` and `debug` commands. -All I'd need to do is add a `test` command, which might look something like -this: - - :::python - def test(): - """Set the target to test.""" - set(fab_hosts=['sjl.webfactional.com']) - set(fab_key_filename='/Users/sjl/.ssh/stevelosh') - set(remote_app_dir = '~/webapps/stevelosh-test/stevelosh') - set(remote_apache_dir = '~/webapps/stevelosh-test/apache2') - -Deploying the test site would then be a simple `fab test deploy` command. - -Why Should You Try This? ------------------------- - -After you've gotten all of this set up the first time it will start saving you -time every time you deploy. It also prevents stupid mistakes like FTP'ing your -files to the wrong directory on the server. It frees you from those headaches -and lets you concentrate on the real work to be done instead of the busywork. -Plus setting it up again for a second project is a breeze after you've done it -once. - -Obviously the exact details of this won't be a perfect fit for everyone. Maybe -you prefer using git and [GitHub][] for version control. I'm sure there's a -similar way to automate the process on that side of the fence. If you decide -to write something about the details of that please let me know and I'll link -it here. - -My hope is that this will at least give you some ideas about saving yourself -some time. If that helps you create better websites, I'll be happy. Please -feel free to find me on [Twitter][twsl] with any questions or thoughts you have! - -[GitHub]: http://github.com/ -[twsl]: {{links.twsl}} - -Update ------- - -It's been over a year since I originally wrote this entry and I've learned -quite a bit since then. I don't have the time right now to go back and rewrite -the entire article, but here are some of the main things that have changed: - -* I now use [virtualenv and pip][venv] when deploying Django sites. It - sandboxes each site very nicely and makes it easy to deploy. - -[venv]: http://clemesha.org/blog/2009/jul/05/modern-python-hacker-tools-virtualenv-fabric-pip/ - -* I no longer go "through" BitBucket when deploying -- I push directly from my - local machine to the server. This eliminates an extra step, and I can always - push to BitBucket once I'm done with a series of changes. - -* Fabric is completely different. "Ownership/maintenance" of the project has - transferred to a new person and the format of fabfiles has drastically - changed. - -Here's a sample fabfile from one of the projects I'm working on now. There are -no comments, but I think most things should be self-explanitory. If you have -any questions just find me on [Twitter][twsl] and I'll be glad to answer them. - - from fabric.api import * - - REMOTE_HG_PATH = '/home/sjl/bin/hg' - - def prod(): - """Set the target to production.""" - env.hosts = ['sjl.webfactional.com'] - env.remote_app_dir = 'webapps/SAMPLE/SAMPLE' - env.remote_apache_dir = '~/webapps/SAMPLE/apache2' - env.local_settings_file = 'local_settings-prod.py' - env.remote_push_dest = 'ssh://webf/%s' % env.remote_app_dir - env.tag = 'production' - env.venv_name = 'SAMPLE_prod' - - def test(): - """Set the target to test.""" - env.hosts = ['sjl.webfactional.com'] - env.remote_app_dir = 'webapps/SAMPLE_test/lindyhub' - env.remote_apache_dir = '~/webapps/SAMPLE_test/apache2' - env.local_settings_file = 'local_settings-test.py' - env.remote_push_dest = 'ssh://webf/%s' % env.remote_app_dir - env.tag = 'test' - env.venv_name = 'SAMPLE_test' - - def deploy(): - """Deploy the site. - - This will also add a local Mercurial tag with the name of the environment - (test or production) to your the local repository, and then sync it to - the remote repo as well. - - This is nice because you can easily see which changeset is currently - deployed to a particular environment in 'hg glog'. - """ - require('hosts', provided_by=[prod, test]) - require('remote_app_dir', provided_by=[prod, test]) - require('remote_apache_dir', provided_by=[prod, test]) - require('local_settings_file', provided_by=[prod, test]) - require('remote_push_dest', provided_by=[prod, test]) - require('tag', provided_by=[prod, test]) - require('venv_name', provided_by=[prod, test]) - - local("hg tag --local --force %s" % env.tag) - local("hg push %s --remotecmd %s" % (env.remote_push_dest, REMOTE_HG_PATH)) - put(".hg/localtags", "%s/.hg/localtags" % env.remote_app_dir) - run("cd %s; hg update -C %s" % (env.remote_app_dir, env.tag)) - put("%s" % env.local_settings_file, "%s/local_settings.py" % env.remote_app_dir) - - run("workon %s; cd %s; python manage.py syncdb" % (env.venv_name, env.remote_app_dir)) - restart() - - def restart(): - """Restart apache on the server.""" - require('hosts', provided_by=[prod, test]) - require('remote_apache_dir', provided_by=[prod, test]) - - run("%s/bin/stop; sleep 2; %s/bin/start" % (env.remote_apache_dir, env.remote_apache_dir)) - - def debugon(): - """Turn debug mode on for the server.""" - require('hosts', provided_by=[prod, test]) - require('remote_app_dir', provided_by=[prod, test]) - require('remote_apache_dir', provided_by=[prod, test]) - - run("cd %s; sed -i -e 's/DEBUG = .*/DEBUG = True/' local_settings.py" % env.remote_app_dir) - restart() - - def debugoff(): - """Turn debug mode off for the server.""" - require('hosts', provided_by=[prod, test]) - require('remote_app_dir', provided_by=[prod, test]) - require('remote_apache_dir', provided_by=[prod, test]) - - run("cd %s; sed -i -e 's/DEBUG = .*/DEBUG = False/' local_settings.py" % env.remote_app_dir) - restart() - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2009/01/deploying-site-fabric-and-mercurial.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2009/01/deploying-site-fabric-and-mercurial.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,320 @@ ++++ +title = "Deploying with Fabric & Mercurial" +snip = "Trimming typing." +date = 2009-01-15T20:51:09Z +draft = false + ++++ + +Earlier tonight I added support for the [Mint][] [Bird Feeder][] plugin to my +site's [RSS feeds][]. Bird Feeder isn't designed to work with [Django][] so I +had to change a few things to get it up and running. I mostly followed the +[instructions on +Hicks-Wright.net](http://hicks-wright.net/blog/minty-django-feeds/) with a few +tweaks. + +While I was trying to get things running smoothly I had to redeploy the site a +bunch of times so I could test the changes I made. If I were simply FTP'ing +the files over each time I wanted to redeploy it would have been a huge pain. +Fortunately, I have another way to do it that cuts down on my typing. + +[Mint]: http://haveamint.com +[Bird Feeder]: http://haveamint.com/peppermill/pepper/11/bird_feeder/ +[RSS feeds]: /rss/ +[Django]: {{links.django}} + +{{% toc %}} + +My Basic Deployment Steps +------------------------- + +The code for my site is stored in a Mercurial repository. There are actually +three copies of the repository that I use: + +* One is on my local machine, which I commit to as I make changes. +* One is on [BitBucket][], which I use to share the code with the public. +* One is on my host's webserver, and is what actually gets served as the website. + +When I'm ready to deploy a new version of the site, I push the changes I've +made on my local repository to the one on BitBucket, then pull the changes +from BitBucket down to the server. Using push/pull means I don't have to worry +about transferring the files myself. Using BitBucket in the middle (instead of +going right from my local machine to the server) means I don't have to worry +about serving either repository myself and dealing with port forwarding or +security issues. + +[BitBucket]: http://bitbucket.org + +Putting BitBucket in the Middle +------------------------------- + +Using BitBucket as an intermediate repository is actually fairly simple. Here +are the basic steps I use to get a project up and running like this. + +First, create a new repository on BitBucket. Make sure the name is what you +want. Feel free to make it private if you just want it for this, or public if +you want to [go open source][]. + +[go open source]: /blog/entry/2009/1/13/going-open-source/ + +### Set Up Your Local Machine + +Clone this new, empty repository to your local machine. If you already have +code written you'll need to copy it into this folder after cloning. I don't +know if there's a way to push a brand-new repository to BitBucket; if you know +how please tell me. + +On your local machine, edit the `.hg/hgrc` file in the repository and change +the default path to: + +```ini +default = ssh://hg@bitbucket.org/username/repositoryname/ +``` + +That will let you push/pull to and from BitBucket over SSH. Doing it that way +means you can use public/private key authentication and avoid typing your +password every single time. A fairly comprehensive guide to that can be found +[here](http://www.securityfocus.com/infocus/1810). You can ignore the +server-side configuration; you just need to add your public key on BitBucket's +account settings page and you should be set. + +**UPDATE:** I didn't realize it before, but BitBucket has a [guide to using SSH with BitBucket](http://bitbucket.org/help/using-ssh/). It's definitely worth looking at. + +Now you should be able to use `hg push` and `hg pull` on your local machine to +push and pull changes to and from the BitBucket repository. The next step is +getting it set up on the server side. + +### Set Up Your Server + +On your server, use `hg clone` to clone the BitBucket repository to wherever +you want to serve it from. Edit the `.hg/hgrc` file in that one and change the +default path to the same value as before: + +```ini +default = ssh://hg@bitbucket.org/username/repositoryname/ +``` + +Once again, set up public/private key authentication; this time between the +server and BitBucket. You can either copy your public and private keys from +your local machine to the server (if you trust/own it) or you can create a new +pair and add its public key to your BitBucket account as well. + +While you're at it, set up public/private key authentication to go from your +local machine to your server too. It'll pay off in the long run. + +Now that you've got both sides working, you can develop and deploy like so: + +* Make changes on your local machine, committing as you go. +* Push the changes to BitBucket. +* SSH into your server and pull the changes down. +* Restart the web server process if necessary. + +Not too bad! Instead of manually managing file transfers you can let Mercurial +do it for you. It'll only pull down the files that have changed, and will +always put them in exactly the right spot. That's pretty convenient, but we +can do better. + +Weaving it All Together with Fabric +----------------------------------- + +Being able to push and pull is all well and good, but that's still a lot of +typing. You need to enter a command to push your changes, a command to SSH to +the server, a command to change to the deploy directory, a command to pull the +changes, and a command to restart the server process. That's five commands, +which is four commands too many for me. + +To automate the process, I use the wonderful [Fabric][] tool. If you haven't +seen it before you should take a look. To follow along with the rest of the +section you should read the examples on the site and install Fabric on your +local machine. + +### My Current Setup + +Here's the fabfile I use for deploying my site to my host ([WebFaction][]). +It's pretty specific to my needs but I'm sure it will give you an idea of +where to start. + +[Fabric]: {{links.fabric}} +[WebFaction]: {{links.webfaction}} + +```python +def prod(): + """Set the target to production.""" + set(fab_hosts=['sjl.webfactional.com']) + set(fab_key_filename='/Users/sjl/.ssh/stevelosh') + set(remote_app_dir='~/webapps/stevelosh/stevelosh') + set(remote_apache_dir='~/webapps/stevelosh/apache2') + +def deploy(): + """Deploy the site.""" + require('fab_hosts', provided_by = [prod,]) + local("hg push") + run("cd $(remote_app_dir); hg pull; hg update") + run("cd $(remote_app_dir); python2.5 manage.py syncdb") + run("$(remote_apache_dir)/bin/stop; sleep 1; $(remote_apache_dir)/bin/start") + +def debugon(): + """Turn debug mode on for the production server.""" + require('fab_hosts', provided_by = [prod,]) + run("cd $(remote_app_dir); sed -i -e 's/DEBUG = .*/DEBUG = True/' deploy.py") + run("$(remote_apache_dir)/bin/stop; sleep 1; $(remote_apache_dir)/bin/start") + +def debugoff(): + """Turn debug mode off for the production server.""" + require('fab_hosts', provided_by = [prod,]) + run("cd $(remote_app_dir); sed -i -e 's/DEBUG = .*/DEBUG = False/' deploy.py") + run("$(remote_apache_dir)/bin/stop; sleep 1; $(remote_apache_dir)/bin/start") +``` + +When I'm finished committing to my local repository and I want to deploy the +site, I just use the command `fab prod deploy` on my local machine. Fabric +pushes my local repository to BitBucket, logs into the server (with my public +key—no typing in passwords), pulls down the new changes from BitBucket +and restarts the server process. + +I also set up a couple of debug commands so I can type `fab prod debugon` and +`fab prod debugoff` to change the `settings.DEBUG` option of my Django app. +Sometimes it's useful to turn on debug to find out exactly why a page is +breaking on the server. + +### Extending It + +The reason I split off the `prod` command is so I can set up a separate test +app (on the server) in the future and reuse the `deploy` and `debug` commands. +All I'd need to do is add a `test` command, which might look something like +this: + +```python +def test(): + """Set the target to test.""" + set(fab_hosts=['sjl.webfactional.com']) + set(fab_key_filename='/Users/sjl/.ssh/stevelosh') + set(remote_app_dir = '~/webapps/stevelosh-test/stevelosh') + set(remote_apache_dir = '~/webapps/stevelosh-test/apache2') +``` + +Deploying the test site would then be a simple `fab test deploy` command. + +Why Should You Try This? +------------------------ + +After you've gotten all of this set up the first time it will start saving you +time every time you deploy. It also prevents stupid mistakes like FTP'ing your +files to the wrong directory on the server. It frees you from those headaches +and lets you concentrate on the real work to be done instead of the busywork. +Plus setting it up again for a second project is a breeze after you've done it +once. + +Obviously the exact details of this won't be a perfect fit for everyone. Maybe +you prefer using git and [GitHub][] for version control. I'm sure there's a +similar way to automate the process on that side of the fence. If you decide +to write something about the details of that please let me know and I'll link +it here. + +My hope is that this will at least give you some ideas about saving yourself +some time. If that helps you create better websites, I'll be happy. Please +feel free to find me on [Twitter][twsl] with any questions or thoughts you have! + +[GitHub]: http://github.com/ +[twsl]: {{links.twsl}} + +Update +------ + +It's been over a year since I originally wrote this entry and I've learned +quite a bit since then. I don't have the time right now to go back and rewrite +the entire article, but here are some of the main things that have changed: + +* I now use [virtualenv and pip][venv] when deploying Django sites. It + sandboxes each site very nicely and makes it easy to deploy. + +[venv]: http://clemesha.org/blog/2009/jul/05/modern-python-hacker-tools-virtualenv-fabric-pip/ + +* I no longer go "through" BitBucket when deploying — I push directly from my + local machine to the server. This eliminates an extra step, and I can always + push to BitBucket once I'm done with a series of changes. + +* Fabric is completely different. "Ownership/maintenance" of the project has + transferred to a new person and the format of fabfiles has drastically + changed. + +Here's a sample fabfile from one of the projects I'm working on now. There are +no comments, but I think most things should be self-explanitory. If you have +any questions just find me on [Twitter][twsl] and I'll be glad to answer them. + + from fabric.api import * + + REMOTE_HG_PATH = '/home/sjl/bin/hg' + + def prod(): + """Set the target to production.""" + env.hosts = ['sjl.webfactional.com'] + env.remote_app_dir = 'webapps/SAMPLE/SAMPLE' + env.remote_apache_dir = '~/webapps/SAMPLE/apache2' + env.local_settings_file = 'local_settings-prod.py' + env.remote_push_dest = 'ssh://webf/%s' % env.remote_app_dir + env.tag = 'production' + env.venv_name = 'SAMPLE_prod' + + def test(): + """Set the target to test.""" + env.hosts = ['sjl.webfactional.com'] + env.remote_app_dir = 'webapps/SAMPLE_test/lindyhub' + env.remote_apache_dir = '~/webapps/SAMPLE_test/apache2' + env.local_settings_file = 'local_settings-test.py' + env.remote_push_dest = 'ssh://webf/%s' % env.remote_app_dir + env.tag = 'test' + env.venv_name = 'SAMPLE_test' + + def deploy(): + """Deploy the site. + + This will also add a local Mercurial tag with the name of the environment + (test or production) to your the local repository, and then sync it to + the remote repo as well. + + This is nice because you can easily see which changeset is currently + deployed to a particular environment in 'hg glog'. + """ + require('hosts', provided_by=[prod, test]) + require('remote_app_dir', provided_by=[prod, test]) + require('remote_apache_dir', provided_by=[prod, test]) + require('local_settings_file', provided_by=[prod, test]) + require('remote_push_dest', provided_by=[prod, test]) + require('tag', provided_by=[prod, test]) + require('venv_name', provided_by=[prod, test]) + + local("hg tag --local --force %s" % env.tag) + local("hg push %s --remotecmd %s" % (env.remote_push_dest, REMOTE_HG_PATH)) + put(".hg/localtags", "%s/.hg/localtags" % env.remote_app_dir) + run("cd %s; hg update -C %s" % (env.remote_app_dir, env.tag)) + put("%s" % env.local_settings_file, "%s/local_settings.py" % env.remote_app_dir) + + run("workon %s; cd %s; python manage.py syncdb" % (env.venv_name, env.remote_app_dir)) + restart() + + def restart(): + """Restart apache on the server.""" + require('hosts', provided_by=[prod, test]) + require('remote_apache_dir', provided_by=[prod, test]) + + run("%s/bin/stop; sleep 2; %s/bin/start" % (env.remote_apache_dir, env.remote_apache_dir)) + + def debugon(): + """Turn debug mode on for the server.""" + require('hosts', provided_by=[prod, test]) + require('remote_app_dir', provided_by=[prod, test]) + require('remote_apache_dir', provided_by=[prod, test]) + + run("cd %s; sed -i -e 's/DEBUG = .*/DEBUG = True/' local_settings.py" % env.remote_app_dir) + restart() + + def debugoff(): + """Turn debug mode off for the server.""" + require('hosts', provided_by=[prod, test]) + require('remote_app_dir', provided_by=[prod, test]) + require('remote_apache_dir', provided_by=[prod, test]) + + run("cd %s; sed -i -e 's/DEBUG = .*/DEBUG = False/' local_settings.py" % env.remote_app_dir) + restart() + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2009/01/going-open-source.html --- a/content/blog/2009/01/going-open-source.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,138 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "Going Open Source" - snip: "Why I’m making the code to this website public." - created: 2009-01-13 20:08:56 -%} - -{% block article %} - -Well, I've finally taken the plunge and made this site open source. It's held -in a [Mercurial][] repository, but you can view or download the source code -from without any special tools. - -The site was built with the [Django][] framework. If you want to know more -check out the [project page][]. This post isn't going to duplicate what's -there; instead I want to write about why I made the site open source at all. -There are a couple of main reasons, and not *all* of them are selfish! - -[Mercurial]: {{links.mercurial}} -[Django]: {{links.django}} -[project page]: /projects/stevelosh-com - -I Get More Traffic ------------------- - -The first reason to make a site open source is to try to get more hits. You -certainly don't *lose* any hits from doing it, and when developers find your -link on places like [DjangoSites][] they might be more inclined to visit if -they knew they could look at the source code later. - -You also have the chance to post on [Twitter][], [Tumblr][], or other places -about the source code of the site. Each of those posts can expose more people -to the website. It's easy to overdo this though. I heartily agree with the -sentiments of [howtousetwitterformarketingandpr.com][htutfmapr] on that issue. - -Don't worry, I won't be obnoxious about it. - -[DjangoSites]: http://djangosites.org/ -[Twitter]: http://twitter.com/ -[Tumblr]: http://tumblr.com/ -[htutfmapr]: http://howtousetwitterformarketingandpr.com - -I Can Give Back to the (Programming) Community ----------------------------------------------- - -Yeah, I know. It sounds cliched, but it's true. - -The open source community has produced a *ridiculous* amount of software, some -of which we use every day. Software like [Apache][], [Django][], and countless -others simply wouldn't exist in anything remotely close to their current -forms. - -Being able to look at the source code for a particular project can be -extremely helpful to someone trying to learn how to do something similar. -Instead of firing up an email client and sending a message to the creator they -can simply look at how it works themselves. They can download it, tweak it, -break it, and figure out what makes it tick. There's no substitute for that. - -I strongly believe that the world would be a better place if everyone shared -their knowledge (in all fields) and we all learned from each other. In some -cases it's simply not practical, but more often than not I think people want -to horde their knowledge and feel superior. Sure, that might be nice for your -ego, but does it really make the world a happier, more beautiful place to -live? - -[Apache]: http://apache.org/ - -I Might Get Free Code! ----------------------- - -Well, probably not, but who knows? Maybe some developer has a lot of free time -on his or her hands? - -Distributed version control systems like [Mercurial][] make it extremely easy -to help out with open source projects. For example, if you were looking -through my code and saw a bug that you wanted to fix, you could do the -following: - -* Clone the repository to your own machine. * Fix the bug in that local copy, -committing to your local repository as you go. * Go back to the website and -click the "Pull Request" button. * I review the changes. * If I like them I -press a button to merge them into the main repository. - -Once the changes are in the main repository I just run a single script to -redeploy the site as usual and the changes are live. It's really an almost -painless process. Compare that to something like Subversion or (ick) CVS: - -* Check out the repository to your own machine. -* Fix the bug in that local copy, without committing (don't mess up!). -* Create a patch using diff or something similar. -* Email me the patch. -* I review the changes. -* If I like them I check out a fresh copy of the repository. -* I apply the patch to this copy. -* I merge the patch back into the main repository. - -It's much more difficult because I don't want to just give any anonymous -person commit access to my repository. - -It Makes Me Fix My Damn Code ----------------------------- - -This is probably the most tangible and important benefit of my open sourcing -this site. It forced me to rewrite parts of the code that weren't secure and -do it the right way. - -Passwords stored in source control? Yeah, *definitely* a bad thing. - -Of course, it *was* kind of a pain in the ass to refactor a bunch of stuff. -Really, though, the main problem I had was my own procrastination and going -open source was a good nudge. Plus when I start work on another site I'll know -how to do it right the first time. - -Aside from major security and privacy issues, it also gives me a big incentive -to clean up my code and make it more elegant. Clean, elegant code makes me -happy. It's less likely to have bugs (because there are fewer places for them -to hide) and it's far easier to maintain in the long run. - -It's also simply more fun to read and work with. - -You Should Probably Try It --------------------------- - -Obviously not all software is appropriate for open sourcing, but if you've got -a pet project that you've been working and you think it might benefit from a -few more sets of eyes, try it! - -You don't need to spend money or configure much of anything to share your -code; sites like [BitBucket][] and [GitHub][] offer free hosting and great -interfaces for repositories of open source code. - -If you have any questions or comments I'd love to hear them! - -[BitBucket]: http://bitbucket.org/ -[GitHub]: http://github.com/ - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2009/01/going-open-source.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2009/01/going-open-source.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,135 @@ ++++ +title = "Going Open Source" +snip = "Why I’m making the code to this website public." +date = 2009-01-13T20:08:56Z +draft = false + ++++ + +Well, I've finally taken the plunge and made this site open source. It's held +in a [Mercurial][] repository, but you can view or download the source code +from without any special tools. + +The site was built with the [Django][] framework. If you want to know more +check out the [project page][]. This post isn't going to duplicate what's +there; instead I want to write about why I made the site open source at all. +There are a couple of main reasons, and not *all* of them are selfish! + +[Mercurial]: {{links.mercurial}} +[Django]: {{links.django}} +[project page]: /projects/stevelosh-com + +I Get More Traffic +------------------ + +The first reason to make a site open source is to try to get more hits. You +certainly don't *lose* any hits from doing it, and when developers find your +link on places like [DjangoSites][] they might be more inclined to visit if +they knew they could look at the source code later. + +You also have the chance to post on [Twitter][], [Tumblr][], or other places +about the source code of the site. Each of those posts can expose more people +to the website. It's easy to overdo this though. I heartily agree with the +sentiments of [howtousetwitterformarketingandpr.com][htutfmapr] on that issue. + +Don't worry, I won't be obnoxious about it. + +[DjangoSites]: http://djangosites.org/ +[Twitter]: http://twitter.com/ +[Tumblr]: http://tumblr.com/ +[htutfmapr]: http://howtousetwitterformarketingandpr.com + +I Can Give Back to the (Programming) Community +---------------------------------------------- + +Yeah, I know. It sounds cliched, but it's true. + +The open source community has produced a *ridiculous* amount of software, some +of which we use every day. Software like [Apache][], [Django][], and countless +others simply wouldn't exist in anything remotely close to their current +forms. + +Being able to look at the source code for a particular project can be +extremely helpful to someone trying to learn how to do something similar. +Instead of firing up an email client and sending a message to the creator they +can simply look at how it works themselves. They can download it, tweak it, +break it, and figure out what makes it tick. There's no substitute for that. + +I strongly believe that the world would be a better place if everyone shared +their knowledge (in all fields) and we all learned from each other. In some +cases it's simply not practical, but more often than not I think people want +to horde their knowledge and feel superior. Sure, that might be nice for your +ego, but does it really make the world a happier, more beautiful place to +live? + +[Apache]: http://apache.org/ + +I Might Get Free Code! +---------------------- + +Well, probably not, but who knows? Maybe some developer has a lot of free time +on his or her hands? + +Distributed version control systems like [Mercurial][] make it extremely easy +to help out with open source projects. For example, if you were looking +through my code and saw a bug that you wanted to fix, you could do the +following: + +* Clone the repository to your own machine. * Fix the bug in that local copy, +committing to your local repository as you go. * Go back to the website and +click the "Pull Request" button. * I review the changes. * If I like them I +press a button to merge them into the main repository. + +Once the changes are in the main repository I just run a single script to +redeploy the site as usual and the changes are live. It's really an almost +painless process. Compare that to something like Subversion or (ick) CVS: + +* Check out the repository to your own machine. +* Fix the bug in that local copy, without committing (don't mess up!). +* Create a patch using diff or something similar. +* Email me the patch. +* I review the changes. +* If I like them I check out a fresh copy of the repository. +* I apply the patch to this copy. +* I merge the patch back into the main repository. + +It's much more difficult because I don't want to just give any anonymous +person commit access to my repository. + +It Makes Me Fix My Damn Code +---------------------------- + +This is probably the most tangible and important benefit of my open sourcing +this site. It forced me to rewrite parts of the code that weren't secure and +do it the right way. + +Passwords stored in source control? Yeah, *definitely* a bad thing. + +Of course, it *was* kind of a pain in the ass to refactor a bunch of stuff. +Really, though, the main problem I had was my own procrastination and going +open source was a good nudge. Plus when I start work on another site I'll know +how to do it right the first time. + +Aside from major security and privacy issues, it also gives me a big incentive +to clean up my code and make it more elegant. Clean, elegant code makes me +happy. It's less likely to have bugs (because there are fewer places for them +to hide) and it's far easier to maintain in the long run. + +It's also simply more fun to read and work with. + +You Should Probably Try It +-------------------------- + +Obviously not all software is appropriate for open sourcing, but if you've got +a pet project that you've been working and you think it might benefit from a +few more sets of eyes, try it! + +You don't need to spend money or configure much of anything to share your +code; sites like [BitBucket][] and [GitHub][] offer free hosting and great +interfaces for repositories of open source code. + +If you have any questions or comments I'd love to hear them! + +[BitBucket]: http://bitbucket.org/ +[GitHub]: http://github.com/ + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2009/01/site-redesign.html --- a/content/blog/2009/01/site-redesign.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,47 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "Site Redesign" - snip: "Yeah, I know. Again." - created: 2009-01-11 17:58:23 -%} - -{% block article %} - -Well, I've redesigned the site again. This time it's all me. I built the site -from the ground up with [Django][] and [Python][]. Why? Several reasons: - -* I already pay for hosting at [WebFaction][] for other sites so also paying - [Squarespace][] for my personal one was too wasteful. -* I wanted to practice designing another site with [Django][]. -* I wanted more flexibility than [Squarespace][] supports and I'm not afraid - to get my hands dirty with code. - -If you want to know more about how I created the site, check out the site's -[project page][]. - -Moving over ------------ - -I've tried to make transitioning to the new site as easy as possible. For -example, I set up the urls so that links to the entries on the old blog will -redirect to entries on the new one if they exist. - -Coming soon ------------ - -I'm going to work on porting over more of the old blog entries in the next day -or two. I don't want to bring them all over; just the best ones. - -I'm also going to implement an RSS feed that will encompass projects and blog -entries. - -Let me know what you think! - -[Python]: {{links.python}} -[Django]: {{links.django}} -[WebFaction]: {{links.webfaction}} -[Squarespace]: http://www.squarespace.com/ -[project page]: /projects/stevelosh-com - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2009/01/site-redesign.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2009/01/site-redesign.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,44 @@ ++++ +title = "Site Redesign" +snip = "Yeah, I know. Again." +date = 2009-01-11T17:58:23Z +draft = false + ++++ + +Well, I've redesigned the site again. This time it's all me. I built the site +from the ground up with [Django][] and [Python][]. Why? Several reasons: + +* I already pay for hosting at [WebFaction][] for other sites so also paying + [Squarespace][] for my personal one was too wasteful. +* I wanted to practice designing another site with [Django][]. +* I wanted more flexibility than [Squarespace][] supports and I'm not afraid + to get my hands dirty with code. + +If you want to know more about how I created the site, check out the site's +[project page][]. + +Moving over +----------- + +I've tried to make transitioning to the new site as easy as possible. For +example, I set up the urls so that links to the entries on the old blog will +redirect to entries on the new one if they exist. + +Coming soon +----------- + +I'm going to work on porting over more of the old blog entries in the next day +or two. I don't want to bring them all over; just the best ones. + +I'm also going to implement an RSS feed that will encompass projects and blog +entries. + +Let me know what you think! + +[Python]: {{links.python}} +[Django]: {{links.django}} +[WebFaction]: {{links.webfaction}} +[Squarespace]: http://www.squarespace.com/ +[project page]: /projects/stevelosh-com + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2009/02/how-and-why-i-dj.html --- a/content/blog/2009/02/how-and-why-i-dj.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,212 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "How & Why I DJ" - snip: "I like playing music for dancers." - created: 2009-02-06 17:53:44 -%} - -{% block article %} - -I've been DJ'ing at swing and blues dances for a while now. It's extremely fun -and rewarding. I've got a system that I use to keep myself sane while doing it -and I felt like sharing. Let me know what you think! - -[TOC] - -## How I Started - -When I was just starting to get really interested in dancing I didn't think -I'd want to DJ much. I thought it would be a lot of work and that I'd hate -having to stay near a computer during a dance. It just didn't seem like very -much fun. - -The only reason I began to try my hand at it was because we needed to have -good music to play at RIT Swing Dance Club when my old roommates and I started -teaching there. I wanted the other people to have a good time and hear the -awesome music that we love dancing to. I figured that it didn't need to be a -big deal; I'd just find some cool music and play it at club. - -Once I started, I realized that I really like being able to play great music -for other people to dance to. It sounds cliche, but it really *is* rewarding. -There's something special about having a kind of control over an entire dance -floor. - -After I became comfortable DJ'ing at club it was only a matter of time before -I started [signing up to DJ][ljdj] at [Lindy Jam][] and then playing music at -our little blues parties. I love it. DJ'ing is good for me — it forces -me to look for interesting new music and really *listen* to it to hear how it -works. Really listening (without dancing) closely to swing or blues music is -something that I don't think enough dancers do. - -## Where Do I Find Music? - -Most of the music I play comes from [eMusic][]. It's a great site — you -pay $15 and get about 40 tracks per month. That's far cheaper than iTunes and -it's all DRM-free mp3 files. They have a great collection of jazz and blues -(and a lot of indie and classical music too). I highly recommend them if -you're looking for some awesome music. - -Previously I've wanted nothing to do with iTunes because of the DRM. -Thankfully Apple has announced that they've convinced the record labels to -allow DRM-free downloads of everything. I'll still get most of my music from -eMusic (it's cheaper) but if they don't have a song I want I'll definitely -look on iTunes. - -As for CDs and vinyl, I do occasionally buy and rip some, but for the most -part digital music is simply faster, easier, and cheaper to use. - -## Sorting My Music - -Over the past couple of years I've turned into a pretty organized person. If I -don't have a good method of sorting and organizing my music I'll go crazy when -I try to DJ. I've developed a way of organizing everything over the years I've -been DJ'ing which seems to work pretty well for me. - -### "Danceable" Music - -The first thing I do when I download a new CD from eMusic is skim each track -and decide if it's even remotely danceable. If it's got a beat I can dance to -at all, it goes into my "Lindy & Blues" playlist in iTunes. I don't worry -about how good the song is — the important part for me is getting the -decent songs into that playlist so I don't forget about them. - -### Tempo - -My next step is tagging danceable songs with BPM (beats per minute) -information. This is really nice to know when you're out at a dane and -thinking: "I should play a song that's a bit faster than that last one." - -I've experimented with a couple of different methods in the past but they all -seemed pretty cumbersome. The solution came in the form of this [Tempo -Widget][] for my Mac's dashboard. Click the button on each beat, double click -the numbers to save the BPM to the currently playing iTunes song. - -That's nice, but it's not enough for a nerd like me. The mouse is so slow; -what I really needed was a way to use the keyboard for everything so I could -fly through tagging my songs in less than 15 seconds each. I opened up the -code for the widget and added a couple things. I can press any key on a beat -instead of clicking the button, and I can hit Cmd+S to save the BPM to the -iTunes song instead of double clicking the numbers. - -I also use [Quicksilver][] to add keyboard shortcuts to move to the -next/previous song in iTunes. Now that I've got all that done, when I want to -tag a bunch of new songs with BPMs I can do this: - -1. Start playing the first song and open up the dashboard. -2. Press any key on each beat until I have a consistent BPM displaying. -3. Press Cmd+S to save the BPM to the song. -4. Press Cmd+Option+Ctrl+Right Arrow to move to the next song. -5. Repeat steps 2-5 for each song. - -It usually takes me 15 seconds or less per song and saves me a ton of time (I -have about 900 songs tagged with BPMs right now!). If you're interested in my -modified version of the Tempo Widget let me know. - -### Rating (Smartly) - -Let's face it: some songs are better than others. Some songs are fun to dance -to, but others almost *force* you out of your seat. I like to be able to -glance at my playlist and have a rough idea of which *really awesome* songs -I've got at a specific tempo. - -Some people might say: "You should know all of your music by heart and -shouldn't need to rely on ratings." I disagree — maybe I'm just not as -diligent in studying but there's no way I could remember every single song in -my playlist. Of course you should know your music, but that doesn't mean you -can't use whatever tools you like to make your DJ'ing even more effective. -Ratings help me remember to play a song that I might otherwise have forgotten -about and let me see what my options are if I really need a fantastic song -*right now*. - -Unfortunately I didn't start rating my danceable songs until a month or two -ago and so I'm only about halfway through. One thing that helps is iTunes' -Smart Playlists. I can create a smart playlist that only shows me songs that -are in my "Lindy & Blues" playlist but aren't yet rated. - -![Unrated Smart Playlist Screenshot](/media/images{{parent_url}}/dj-playlist-unrated.png "Unrated Smart Playlist") - -I have an iPod as well, so when I'm out and listening to my music I can set -the rating right on the iPod. When I get back to my computer I plug in the -iPod and it syncs the ratings back to iTunes for me. - -## At a Dance - -All of the steps I've talked about so far have one goal: to make playing great -music at an actual dance easier. - -### Previewing - -I rarely preview songs before I play them; I know my collection pretty well -and usually have a good idea whether or not a song will fit. Still, it's -definitely nice to be able to hear a song now and then. - -Some DJ's really love external sound cards which let you have two music -players open on your machine; one plays the music going to the room speakers -and the other plays into a set of headphones. I don't use this setup for a few -reasons: - -* I can't be bothered to buy an external sound card. I know, I'm lazy. -* Computers are tricky, and adding another sound card into the mix makes - things trickier. Having another audio interface often causes some headaches - when you're ready to DJ and no sound is coming out of one port because - there's a configuration option messed up somewhere. -* I don't trust myself. I can easily see myself accidentally trying to preview - on the music player that's going to the room. - -What do I do instead? I use my iPod. Sure, the interface isn't as easy to -navigate as a music player on a laptop, but it doesn't cost any extra money (I -have it anyway) and there's no chance of accidentally messing up the currently -playing song. - -### Running Playlists - -Throughout the night as I decide which songs I want to play I put them in two -separate lists. First they go into my "Short List" which is usually around -five to ten songs that I think I'll want to play fairly soon. That gives me -easy access to songs I'll probably play soon. - -The "Running" playlist is the one that's actually playing through the -speakers. I usually have one or two songs queued up ahead of time in case I -get distracted from my laptop for a bit. - -### Sorting on the Fly - -All the work I put into tagging and rating my music pays off when I'm actually -trying to decide what to play next. When I'm looking through my collection my -iTunes window will usually look like this (except for being slightly bigger -and having the sidebar showing): - -![Sorted Danceable Playlist Screenshot](/media/images{{parent_url}}/dj-playlist-sorting.png "Sorted Danceable Playlist") - -I can use the search bar in the upper right if I'm looking for something -specific. I can sort by BPM if I know I want a song at a rough tempo. The -ratings draw my eyes to the songs that I love the most. The comments are there -to remind me about songs that have long intro/outros, double timed sections, -etc. Having all of that at my fingers makes it much easier for me to find -exactly what I want and get it to the speakers. - -## And So... - -I wrote this because I wanted to share how and why I DJ for dancers. If you're -a DJ, I hope I've given you a couple of ideas for new tricks to make your job -easier. **Let me know what you think on [Twitter][twsl]!** - -If you don't DJ but think you might want to, great! Please don't be scared off -by my absurdly organized workflow — just because it works for me doesn't -mean you have to do it. Find some good music and [sign up to DJ at Lindy -Jam][ljdj]! If you think two hours is too long for you as when you're first -learning to DJ just ask myself, Matt, Jesse, Mike or Beth if we'll share the -night half & half with you. If you have any questions please let me know! - - -[Lindy Jam]: http://lindyjam.com/ -[ljdj]: http://lindyjam.com/schedule/ -[eMusic]: http://emusic.com/ -[Quicksilver]: http://www.blacktree.com/ -[Tempo Widget]: http://mac.softpedia.com/get/Dashboard-Widgets/Music/Tempo-Widget.shtml -[twsl]: {{links.twsl}} - - - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2009/02/how-and-why-i-dj.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2009/02/how-and-why-i-dj.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,209 @@ ++++ +title = "How & Why I DJ" +snip = "I like playing music for dancers." +date = 2009-02-06T17:53:44Z +draft = false + ++++ + +I've been DJ'ing at swing and blues dances for a while now. It's extremely fun +and rewarding. I've got a system that I use to keep myself sane while doing it +and I felt like sharing. Let me know what you think! + +{{% toc %}} + +## How I Started + +When I was just starting to get really interested in dancing I didn't think +I'd want to DJ much. I thought it would be a lot of work and that I'd hate +having to stay near a computer during a dance. It just didn't seem like very +much fun. + +The only reason I began to try my hand at it was because we needed to have +good music to play at RIT Swing Dance Club when my old roommates and I started +teaching there. I wanted the other people to have a good time and hear the +awesome music that we love dancing to. I figured that it didn't need to be a +big deal; I'd just find some cool music and play it at club. + +Once I started, I realized that I really like being able to play great music +for other people to dance to. It sounds cliche, but it really *is* rewarding. +There's something special about having a kind of control over an entire dance +floor. + +After I became comfortable DJ'ing at club it was only a matter of time before +I started [signing up to DJ][ljdj] at [Lindy Jam][] and then playing music at +our little blues parties. I love it. DJ'ing is good for me — it forces +me to look for interesting new music and really *listen* to it to hear how it +works. Really listening (without dancing) closely to swing or blues music is +something that I don't think enough dancers do. + +## Where Do I Find Music? + +Most of the music I play comes from [eMusic][]. It's a great site — you +pay $15 and get about 40 tracks per month. That's far cheaper than iTunes and +it's all DRM-free mp3 files. They have a great collection of jazz and blues +(and a lot of indie and classical music too). I highly recommend them if +you're looking for some awesome music. + +Previously I've wanted nothing to do with iTunes because of the DRM. +Thankfully Apple has announced that they've convinced the record labels to +allow DRM-free downloads of everything. I'll still get most of my music from +eMusic (it's cheaper) but if they don't have a song I want I'll definitely +look on iTunes. + +As for CDs and vinyl, I do occasionally buy and rip some, but for the most +part digital music is simply faster, easier, and cheaper to use. + +## Sorting My Music + +Over the past couple of years I've turned into a pretty organized person. If I +don't have a good method of sorting and organizing my music I'll go crazy when +I try to DJ. I've developed a way of organizing everything over the years I've +been DJ'ing which seems to work pretty well for me. + +### "Danceable" Music + +The first thing I do when I download a new CD from eMusic is skim each track +and decide if it's even remotely danceable. If it's got a beat I can dance to +at all, it goes into my "Lindy & Blues" playlist in iTunes. I don't worry +about how good the song is — the important part for me is getting the +decent songs into that playlist so I don't forget about them. + +### Tempo + +My next step is tagging danceable songs with BPM (beats per minute) +information. This is really nice to know when you're out at a dane and +thinking: "I should play a song that's a bit faster than that last one." + +I've experimented with a couple of different methods in the past but they all +seemed pretty cumbersome. The solution came in the form of this [Tempo +Widget][] for my Mac's dashboard. Click the button on each beat, double click +the numbers to save the BPM to the currently playing iTunes song. + +That's nice, but it's not enough for a nerd like me. The mouse is so slow; +what I really needed was a way to use the keyboard for everything so I could +fly through tagging my songs in less than 15 seconds each. I opened up the +code for the widget and added a couple things. I can press any key on a beat +instead of clicking the button, and I can hit Cmd+S to save the BPM to the +iTunes song instead of double clicking the numbers. + +I also use [Quicksilver][] to add keyboard shortcuts to move to the +next/previous song in iTunes. Now that I've got all that done, when I want to +tag a bunch of new songs with BPMs I can do this: + +1. Start playing the first song and open up the dashboard. +2. Press any key on each beat until I have a consistent BPM displaying. +3. Press Cmd+S to save the BPM to the song. +4. Press Cmd+Option+Ctrl+Right Arrow to move to the next song. +5. Repeat steps 2-5 for each song. + +It usually takes me 15 seconds or less per song and saves me a ton of time (I +have about 900 songs tagged with BPMs right now!). If you're interested in my +modified version of the Tempo Widget let me know. + +### Rating (Smartly) + +Let's face it: some songs are better than others. Some songs are fun to dance +to, but others almost *force* you out of your seat. I like to be able to +glance at my playlist and have a rough idea of which *really awesome* songs +I've got at a specific tempo. + +Some people might say: "You should know all of your music by heart and +shouldn't need to rely on ratings." I disagree — maybe I'm just not as +diligent in studying but there's no way I could remember every single song in +my playlist. Of course you should know your music, but that doesn't mean you +can't use whatever tools you like to make your DJ'ing even more effective. +Ratings help me remember to play a song that I might otherwise have forgotten +about and let me see what my options are if I really need a fantastic song +*right now*. + +Unfortunately I didn't start rating my danceable songs until a month or two +ago and so I'm only about halfway through. One thing that helps is iTunes' +Smart Playlists. I can create a smart playlist that only shows me songs that +are in my "Lindy & Blues" playlist but aren't yet rated. + +![Unrated Smart Playlist Screenshot](/media/images/blog/2009/02/dj-playlist-unrated.png "Unrated Smart Playlist") + +I have an iPod as well, so when I'm out and listening to my music I can set +the rating right on the iPod. When I get back to my computer I plug in the +iPod and it syncs the ratings back to iTunes for me. + +## At a Dance + +All of the steps I've talked about so far have one goal: to make playing great +music at an actual dance easier. + +### Previewing + +I rarely preview songs before I play them; I know my collection pretty well +and usually have a good idea whether or not a song will fit. Still, it's +definitely nice to be able to hear a song now and then. + +Some DJ's really love external sound cards which let you have two music +players open on your machine; one plays the music going to the room speakers +and the other plays into a set of headphones. I don't use this setup for a few +reasons: + +* I can't be bothered to buy an external sound card. I know, I'm lazy. +* Computers are tricky, and adding another sound card into the mix makes + things trickier. Having another audio interface often causes some headaches + when you're ready to DJ and no sound is coming out of one port because + there's a configuration option messed up somewhere. +* I don't trust myself. I can easily see myself accidentally trying to preview + on the music player that's going to the room. + +What do I do instead? I use my iPod. Sure, the interface isn't as easy to +navigate as a music player on a laptop, but it doesn't cost any extra money (I +have it anyway) and there's no chance of accidentally messing up the currently +playing song. + +### Running Playlists + +Throughout the night as I decide which songs I want to play I put them in two +separate lists. First they go into my "Short List" which is usually around +five to ten songs that I think I'll want to play fairly soon. That gives me +easy access to songs I'll probably play soon. + +The "Running" playlist is the one that's actually playing through the +speakers. I usually have one or two songs queued up ahead of time in case I +get distracted from my laptop for a bit. + +### Sorting on the Fly + +All the work I put into tagging and rating my music pays off when I'm actually +trying to decide what to play next. When I'm looking through my collection my +iTunes window will usually look like this (except for being slightly bigger +and having the sidebar showing): + +![Sorted Danceable Playlist Screenshot](/media/images/blog/2009/02/dj-playlist-sorting.png "Sorted Danceable Playlist") + +I can use the search bar in the upper right if I'm looking for something +specific. I can sort by BPM if I know I want a song at a rough tempo. The +ratings draw my eyes to the songs that I love the most. The comments are there +to remind me about songs that have long intro/outros, double timed sections, +etc. Having all of that at my fingers makes it much easier for me to find +exactly what I want and get it to the speakers. + +## And So... + +I wrote this because I wanted to share how and why I DJ for dancers. If you're +a DJ, I hope I've given you a couple of ideas for new tricks to make your job +easier. **Let me know what you think on [Twitter][twsl]!** + +If you don't DJ but think you might want to, great! Please don't be scared off +by my absurdly organized workflow — just because it works for me doesn't +mean you have to do it. Find some good music and [sign up to DJ at Lindy +Jam][ljdj]! If you think two hours is too long for you as when you're first +learning to DJ just ask myself, Matt, Jesse, Mike or Beth if we'll share the +night half & half with you. If you have any questions please let me know! + + +[Lindy Jam]: http://lindyjam.com/ +[ljdj]: http://lindyjam.com/schedule/ +[eMusic]: http://emusic.com/ +[Quicksilver]: http://www.blacktree.com/ +[Tempo Widget]: http://mac.softpedia.com/get/Dashboard-Widgets/Music/Tempo-Widget.shtml +[twsl]: {{links.twsl}} + + + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2009/02/how-i-shoot-dances.html --- a/content/blog/2009/02/how-i-shoot-dances.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,371 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "How I Shoot Dances" - snip: "Slow shutter and flash." - created: 2009-02-09 18:04:36 -%} - -{% block article %} - -Last weekend [Gordon Webster][] came to Rochester, NY to play a dance at The -Keg. It was awesome (of course) and everyone had a great time. - -I took a lot of photos during the course of the night (around 360 actually) -and some of them turned out pretty nice. A few people have wondered how I got -the look I did, so I decided to write a post about it. - -I'm assuming you know what basic terms like aperture and shutter speed mean. -If not, look for a post over at [LindyBloggers][] fairly soon about shooting -dances with a point-and-shoot camera. - -[Gordon Webster]: http://webster.suresong.com/ -[LindyBloggers]: http://lindybloggers.com/ - -[TOC] - -Shooting Dancers is Tough -------------------------- - -One of the reasons I got into photography seriously was so that I could shoot -dancers and musicians. I've since branched out and become interested in a lot -of other aspects of photography, but dancing is a huge part of my life and -photographing dancers is still something I absolutely love. - -I (like many others) thought that getting a DSLR would let me take awesome -dancing photos with a bit of practice. Well, it's not that simple. Dances are -one of the hardest things I shoot for a couple of reasons. - -### We Move - -Dancing, especially in a dance like Lindy Hop, involves a lot of movement. It -makes focusing a huge pain unless you've got *really* good eyes. I don't. When -I first started out the majority (at least 70%) of my dance photos were out of -focus. It's much more difficult than shooting a still (or even slowly moving) -target. - -Moving quickly also means you need a fast shutter speed if you want anything -to be sharp. You can focus perfectly but that won't freeze any movement. This -wouldn't be an issue if it wasn't for the second main problem. - -### The Lighting at Dances is Awful - -Seriously, it's terrible. It's always fairly dark to set the mood; you don't -want to dance in a brightly lit place unless you're competing. To make matters -worse, it's almost always ridiculously uneven. Part of the room with usually -be two or three stops darker than the rest. - -Only shooting in one place helps, but is fairy boring. A lot of people will -stay in the same general area for most of the night (damn cliques!) and so if -you only shoot one section of the floor you'll miss a lot of people. - -What Doesn't Work (for Me (Usually)) ------------------------------------- - -I've tried a couple of different techniques to overcome these problems. Some -of them work in some situations but most of the time they leave me unhappy. - - - GirlJamSunday-5512 - - -### Fast Primes, High ISO - -The first remedy I turned to was buying a fast prime lens (Pentax 50mm f/1.4) -so that it would let in more light. With that lens I would turn up the ISO to -1600 or so to get even more sensitivity and hope for the best. This kind of -works, but has some issues. - -#### The Good - -Shooting with a fast prime means I only have to carry one lens around and -don't need to change it during the night. They're usually pretty light too, so -they're easier to work with. 50mm is a nice focal length that lets you stand -far enough away to not get kicked. You also don't need to annoy people with -flash. - -The wide apertures let more light in so you can actually get some decently -exposed photos at workable shutter speeds. The ISO isn't much of an problem as -long as you expose the photo right (if you have to bring up the exposure more -than a half stop in post it looks terrible). - -This solution really excels if you're shooting classes or workshops. The light -there is usually much better than at actual dances but still not ideal. The -wide apertures let you soak up all of that light and you can get some really -nice photos. The example in this section was taken at Girl Jam last year -during one of the classes with my 50mm. - -#### The Bad - -Focusing on moving dancers is hard enough at "normal" apertures; trying to -nail the focus when you've only got six inches of depth of field at f/1.4 is -*nearly fucking impossible*. Maybe other photographers are better at manually -focusing or have amazing Canon/Nikon cameras that can focus on a black cat in -a darkroom, but I'm not and I don't. - -The focal length of most fast primes is generally around 50mm. This isn't too -bad, but I've grown to really love wide angle lenses. 50mm feels too far and -detached for my taste. Yours may be different, so give it a try. - - - LindyJam-0095 - - -### Bounced Flash, Max Sync Speed - -For a while I was adamant that I would never use flash. I figured it would be -annoying and that the "unnatural" light from the flash would somehow look -wrong. I avoided it for a while in favor of fast primes and "natural" light. - -Eventually I noticed that some other photographers shot with flash at dances -and it didn't annoy me (while dancing) at all. I decided to give it a shot and -see how it worked, especially since I was getting really into [Strobist][] and -studio lighting at the time. - -[Strobist]: http://strobist.com/ - -The basic idea is that you use an external flash (preferably off camera, -synched with a cord or radio triggers) and bounce the light from the ceiling -to get more even coverage. You set the aperture to something moderate like f/4 -or f/5.6 and the ISO fairly low. You turn the shutter speed down as low as it -will go and still sync up and let the strobe do the hard work. - -#### The Good - -You get sharp photos! What's more, focusing is no longer something you will -curse vehemently! When you're shooting at f/4 or f/5.6 you have a good amount -of depth of field so the focus doesn't have to be absolutely perfect. - -The flash only lasts about 1/1000 of a second. No one moves much in that time, -so you get perfectly sharp photos despite the movement. - -Now that you're not relying on wide apertures you can start using different -focal lengths instead of sticking with the fast prime or two that you have. -You could even use a zoom lens if you don't want to change lenses all the -time. - -#### The Bad - -Bouncing from the ceiling is far, far better than keeping your flash on camera -and pointing it straight at the subjects, but unless the room has obnoxiously -high ceilings you're still going to get a lot of falloff towards the back of -the room. You might not mind; it *does* help isolate the subjects from their -surroundings but I found myself getting tired of it pretty quickly. - -You also have to carry a flash, sync cord or triggers, and extra batteries -around which is kind of a pain. I don't mind, but if you like traveling light -it could bother you. - -This section's example is one of the better results of this technique: Tango -Cafe's ceilings are very high so the light is pretty even everywhere. High -ceilings come at a price, however: you need more power from the flash to -illuminate everything and so can't shoot as fast. - -Rethinking My Approach ----------------------- - -After using these two methods for a while I stopped and looked at my photos. I -wasn't as happy with them as I would have liked. I sat down and asked myself: -"Why is that?" The photos were exposed well and aside from the focusing -mishaps were pretty sharp. What was missing? - - - CIMG0171.JPG - - -Eventually I came up with my answer: "Movement." Lindy Hop is very much about -movement; it's one of the most important parts of the dance, maybe even *the* -most important part. Getting tack sharp photos is great, but it's very hard to -convey movement with them unless you're a much better photographer than I am. - -Most people are not photographers. Most dancers are also not photographers. -The majority of Lindy Hoppers will take photos with small point-and-shoot -cameras, *if* they can drag themselves away from dancing for a little while! - -The result is that we don't usually see many dance photos (because we're too -busy dancing to take them) and the ones that we do see are usually blurry -(because point and shoot cameras in their default modes are just not equipped -to take sharp ones). - - - CIMG0141.JPG - - -Even with their flaws, *we love them* because they're the few photographic -memories we have of some of the best nights of our lives. - -I wanted people to have the same feeling toward my photography as they do -toward these informal snapshots. I wanted to capture the essence of these -pictures we're so grateful for but use my experience to make something even -better. - -I think I've finally figured out how. - -Using Everything ----------------- - -What evokes the feeling of movement in the point-and-shoot shots we look at? I -think it's the blurriness. Our culture is used to looking at photographs and -we know that "blurry photograph" usually means "moving subject" even if we're -not always 100% clear on the physics of it. - - - GW-0453 - - -As photographers, we know what causes it. A subject moving while the shutter -is open produces blur. Longer shutter speeds mean more blur. The little -cameras that most of our memories come from simply don't have the aperture or -sensitivity of our hefty DSLRs – all the poor little things can do is -leave their shutters open a bit longer to get the light they need. That's why -those cameras make blurry photos with their default settings. - -So how can I add some blurriness to my images? Use a slower shutter speed! I -also want to keep some sharpness though, so the dancers are more recognizable. -To do this, I use flash *at the same time*. The flash freezes the subjects -enough to make them look good and then slow shutter speed and ambient light -take over to add some movement. - -### The Technique - -What do you need to do this kind of thing? You can get by with any modern -camera with a built-in flash, but to really have the flexibility that will -make you happy you need the following things: - -* **A camera with a hot shoe or PC jack and manual controls**. I use a Pentax - K20D when I feel like lugging it out and a Canon G10 when I don't. -* **An external flash.** I use an $80 Vivitar 285hv which will work with - anything, so no complaining about how your camera maker only sells $400 - speedlights. -* **A way to sync your flash with the camera.** Some of the more expensive - flashes have a wireless mode. If not, you can buy radio triggers for $60 or - so, or buy a sync cable for $15 to $20. Trust me, having the flash *off* of - the camera makes things so much easier. - - - GW-0290 - - -#### Step 1 – Dial in the Flash - -The first step to getting this look is to figure out what kind of exposure you -need with *the flash alone* to get a well-lit shot. - -Turn your shutter speed down to the fastest it will sync (1/180th is usually -fine) and use trial and error to find a nice combination of aperture, ISO and -flash power. The actual numbers will depend on several things: how low the -ceilings are, how powerful your flash is, etc. - -Aim for the lowest flash power you can while still keeping a good exposure, a -narrow enough aperture to make focusing easy, and modest ISO noise. Once you -figure it out you can probably keep those settings for the rest of the night, -unless the ceilings are higher on one side of the room or something else -equally annoying. - -#### Step 2 – Dial in the Ambient - -This is the step that adds the movement. First, tone down your flash-only -exposure by a half or whole stop. Do this by reducing the ISO, aperture, or -flash power; any of those is fine. - -Now that your photo is underexposed, turn off the flash. It will probably be -completely black now. *Do not touch the ISO or aperture to fix this.* Lengthen -the shutter speed until you're about two stops underexposed; it will be blurry -and dark as hell but this is what you want right now. - - - GW-0580 - - -#### Step 3 – Combine and Adjust - -Turn the flash back on. Do not touch any other settings – flip the -switch on the flash and start shooting. This will let the flash illuminate the -subjects (because you dialed in the power, ISO and aperture before and haven't -changed them) and the ambient fill in and add movement (because you adjusted -the shutter speed). - -Your first few shots will probably be underexposed or overexposed and have too -much or too little blur. Don't worry, it always takes me at least a half hour -to start taking decent photos this way. You really need to play with the -settings as you go to find out what's going to work for the lighting *that -night*. - -#### Things to Watch Out For - -There are a couple of tricky parts to this style of shooting that I'll -mention. First I'll talk about the typical problems you'll see right away. -More than one of these certainly might apply; fix them one at a time. - - - GW-0656 - -* **If the background is dark or there is not enough blur,** you need to use a - longer shutter speed to let more ambient light in. -* **If the dancers are not sharp enough,** you need *more* flash power. Turn - it up a little bit at a time. -* **If people's shirts, faces and limbs are overexposed *and not blurry*,** - you need to turn down the flash. Bring it down a bit by reducing the power, - narrowing the aperture or reducing the ISO. -* **If shirts, faces and limbs are overexposed *and blurry*,** you need to - shorten the shutter speed to let less ambient light in. - -I'm warning you now, fixing one of these will probably fuck something else up. -It can be really infuriating, but if you slow down, stay calm and think -through it logically you'll be able to narrow it down and figure out what you -need. It's a really good feeling when you finally nail it. - -There are two other tricky problems that I'm still working on myself: - -* **Dances are unevenly lit.** When you walk over to the other side of the - room to shoot, you'll have to adjust the shutter speed to compensate for the - different amount of ambient light. You really only have a range of about - half a stop where the ambient gives you the right amount of blur, so you - have to be careful and stay on your toes. -* **Watch where you bounce.** In rooms with low ceilings, where you point the - flash on the ceiling has a huge effect on the lighting. You need to be - conscious of what part of the ceiling you're bouncing from to get good - results. - -Good Luck! ----------- - -I hope this post helped you out. Even if you hate how my photos look and don't -want to make anything like them, at least it will show you what *not* to do. -I'd love to hear comments or advice on what you like or don't like, and what I -could do better. If you've got questions I'll do my best to answer them too! - -Now go dance, have fun, and make beautiful photographs so we can all remember -the fantastic events we make happen! - - - GW-0438 - - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2009/02/how-i-shoot-dances.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2009/02/how-i-shoot-dances.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,368 @@ ++++ +title = "How I Shoot Dances" +snip = "Slow shutter and flash." +date = 2009-02-09T18:04:36Z +draft = false + ++++ + +Last weekend [Gordon Webster][] came to Rochester, NY to play a dance at The +Keg. It was awesome (of course) and everyone had a great time. + +I took a lot of photos during the course of the night (around 360 actually) +and some of them turned out pretty nice. A few people have wondered how I got +the look I did, so I decided to write a post about it. + +I'm assuming you know what basic terms like aperture and shutter speed mean. +If not, look for a post over at [LindyBloggers][] fairly soon about shooting +dances with a point-and-shoot camera. + +[Gordon Webster]: http://webster.suresong.com/ +[LindyBloggers]: http://lindybloggers.com/ + +{{% toc %}} + +Shooting Dancers is Tough +------------------------- + +One of the reasons I got into photography seriously was so that I could shoot +dancers and musicians. I've since branched out and become interested in a lot +of other aspects of photography, but dancing is a huge part of my life and +photographing dancers is still something I absolutely love. + +I (like many others) thought that getting a DSLR would let me take awesome +dancing photos with a bit of practice. Well, it's not that simple. Dances are +one of the hardest things I shoot for a couple of reasons. + +### We Move + +Dancing, especially in a dance like Lindy Hop, involves a lot of movement. It +makes focusing a huge pain unless you've got *really* good eyes. I don't. When +I first started out the majority (at least 70%) of my dance photos were out of +focus. It's much more difficult than shooting a still (or even slowly moving) +target. + +Moving quickly also means you need a fast shutter speed if you want anything +to be sharp. You can focus perfectly but that won't freeze any movement. This +wouldn't be an issue if it wasn't for the second main problem. + +### The Lighting at Dances is Awful + +Seriously, it's terrible. It's always fairly dark to set the mood; you don't +want to dance in a brightly lit place unless you're competing. To make matters +worse, it's almost always ridiculously uneven. Part of the room with usually +be two or three stops darker than the rest. + +Only shooting in one place helps, but is fairy boring. A lot of people will +stay in the same general area for most of the night (damn cliques!) and so if +you only shoot one section of the floor you'll miss a lot of people. + +What Doesn't Work (for Me (Usually)) +------------------------------------ + +I've tried a couple of different techniques to overcome these problems. Some +of them work in some situations but most of the time they leave me unhappy. + + + GirlJamSunday-5512 + + +### Fast Primes, High ISO + +The first remedy I turned to was buying a fast prime lens (Pentax 50mm f/1.4) +so that it would let in more light. With that lens I would turn up the ISO to +1600 or so to get even more sensitivity and hope for the best. This kind of +works, but has some issues. + +#### The Good + +Shooting with a fast prime means I only have to carry one lens around and +don't need to change it during the night. They're usually pretty light too, so +they're easier to work with. 50mm is a nice focal length that lets you stand +far enough away to not get kicked. You also don't need to annoy people with +flash. + +The wide apertures let more light in so you can actually get some decently +exposed photos at workable shutter speeds. The ISO isn't much of an problem as +long as you expose the photo right (if you have to bring up the exposure more +than a half stop in post it looks terrible). + +This solution really excels if you're shooting classes or workshops. The light +there is usually much better than at actual dances but still not ideal. The +wide apertures let you soak up all of that light and you can get some really +nice photos. The example in this section was taken at Girl Jam last year +during one of the classes with my 50mm. + +#### The Bad + +Focusing on moving dancers is hard enough at "normal" apertures; trying to +nail the focus when you've only got six inches of depth of field at f/1.4 is +*nearly fucking impossible*. Maybe other photographers are better at manually +focusing or have amazing Canon/Nikon cameras that can focus on a black cat in +a darkroom, but I'm not and I don't. + +The focal length of most fast primes is generally around 50mm. This isn't too +bad, but I've grown to really love wide angle lenses. 50mm feels too far and +detached for my taste. Yours may be different, so give it a try. + + + LindyJam-0095 + + +### Bounced Flash, Max Sync Speed + +For a while I was adamant that I would never use flash. I figured it would be +annoying and that the "unnatural" light from the flash would somehow look +wrong. I avoided it for a while in favor of fast primes and "natural" light. + +Eventually I noticed that some other photographers shot with flash at dances +and it didn't annoy me (while dancing) at all. I decided to give it a shot and +see how it worked, especially since I was getting really into [Strobist][] and +studio lighting at the time. + +[Strobist]: http://strobist.com/ + +The basic idea is that you use an external flash (preferably off camera, +synched with a cord or radio triggers) and bounce the light from the ceiling +to get more even coverage. You set the aperture to something moderate like f/4 +or f/5.6 and the ISO fairly low. You turn the shutter speed down as low as it +will go and still sync up and let the strobe do the hard work. + +#### The Good + +You get sharp photos! What's more, focusing is no longer something you will +curse vehemently! When you're shooting at f/4 or f/5.6 you have a good amount +of depth of field so the focus doesn't have to be absolutely perfect. + +The flash only lasts about 1/1000 of a second. No one moves much in that time, +so you get perfectly sharp photos despite the movement. + +Now that you're not relying on wide apertures you can start using different +focal lengths instead of sticking with the fast prime or two that you have. +You could even use a zoom lens if you don't want to change lenses all the +time. + +#### The Bad + +Bouncing from the ceiling is far, far better than keeping your flash on camera +and pointing it straight at the subjects, but unless the room has obnoxiously +high ceilings you're still going to get a lot of falloff towards the back of +the room. You might not mind; it *does* help isolate the subjects from their +surroundings but I found myself getting tired of it pretty quickly. + +You also have to carry a flash, sync cord or triggers, and extra batteries +around which is kind of a pain. I don't mind, but if you like traveling light +it could bother you. + +This section's example is one of the better results of this technique: Tango +Cafe's ceilings are very high so the light is pretty even everywhere. High +ceilings come at a price, however: you need more power from the flash to +illuminate everything and so can't shoot as fast. + +Rethinking My Approach +---------------------- + +After using these two methods for a while I stopped and looked at my photos. I +wasn't as happy with them as I would have liked. I sat down and asked myself: +"Why is that?" The photos were exposed well and aside from the focusing +mishaps were pretty sharp. What was missing? + + + CIMG0171.JPG + + +Eventually I came up with my answer: "Movement." Lindy Hop is very much about +movement; it's one of the most important parts of the dance, maybe even *the* +most important part. Getting tack sharp photos is great, but it's very hard to +convey movement with them unless you're a much better photographer than I am. + +Most people are not photographers. Most dancers are also not photographers. +The majority of Lindy Hoppers will take photos with small point-and-shoot +cameras, *if* they can drag themselves away from dancing for a little while! + +The result is that we don't usually see many dance photos (because we're too +busy dancing to take them) and the ones that we do see are usually blurry +(because point and shoot cameras in their default modes are just not equipped +to take sharp ones). + + + CIMG0141.JPG + + +Even with their flaws, *we love them* because they're the few photographic +memories we have of some of the best nights of our lives. + +I wanted people to have the same feeling toward my photography as they do +toward these informal snapshots. I wanted to capture the essence of these +pictures we're so grateful for but use my experience to make something even +better. + +I think I've finally figured out how. + +Using Everything +---------------- + +What evokes the feeling of movement in the point-and-shoot shots we look at? I +think it's the blurriness. Our culture is used to looking at photographs and +we know that "blurry photograph" usually means "moving subject" even if we're +not always 100% clear on the physics of it. + + + GW-0453 + + +As photographers, we know what causes it. A subject moving while the shutter +is open produces blur. Longer shutter speeds mean more blur. The little +cameras that most of our memories come from simply don't have the aperture or +sensitivity of our hefty DSLRs – all the poor little things can do is +leave their shutters open a bit longer to get the light they need. That's why +those cameras make blurry photos with their default settings. + +So how can I add some blurriness to my images? Use a slower shutter speed! I +also want to keep some sharpness though, so the dancers are more recognizable. +To do this, I use flash *at the same time*. The flash freezes the subjects +enough to make them look good and then slow shutter speed and ambient light +take over to add some movement. + +### The Technique + +What do you need to do this kind of thing? You can get by with any modern +camera with a built-in flash, but to really have the flexibility that will +make you happy you need the following things: + +* **A camera with a hot shoe or PC jack and manual controls**. I use a Pentax + K20D when I feel like lugging it out and a Canon G10 when I don't. +* **An external flash.** I use an $80 Vivitar 285hv which will work with + anything, so no complaining about how your camera maker only sells $400 + speedlights. +* **A way to sync your flash with the camera.** Some of the more expensive + flashes have a wireless mode. If not, you can buy radio triggers for $60 or + so, or buy a sync cable for $15 to $20. Trust me, having the flash *off* of + the camera makes things so much easier. + + + GW-0290 + + +#### Step 1 – Dial in the Flash + +The first step to getting this look is to figure out what kind of exposure you +need with *the flash alone* to get a well-lit shot. + +Turn your shutter speed down to the fastest it will sync (1/180th is usually +fine) and use trial and error to find a nice combination of aperture, ISO and +flash power. The actual numbers will depend on several things: how low the +ceilings are, how powerful your flash is, etc. + +Aim for the lowest flash power you can while still keeping a good exposure, a +narrow enough aperture to make focusing easy, and modest ISO noise. Once you +figure it out you can probably keep those settings for the rest of the night, +unless the ceilings are higher on one side of the room or something else +equally annoying. + +#### Step 2 – Dial in the Ambient + +This is the step that adds the movement. First, tone down your flash-only +exposure by a half or whole stop. Do this by reducing the ISO, aperture, or +flash power; any of those is fine. + +Now that your photo is underexposed, turn off the flash. It will probably be +completely black now. *Do not touch the ISO or aperture to fix this.* Lengthen +the shutter speed until you're about two stops underexposed; it will be blurry +and dark as hell but this is what you want right now. + + + GW-0580 + + +#### Step 3 – Combine and Adjust + +Turn the flash back on. Do not touch any other settings – flip the +switch on the flash and start shooting. This will let the flash illuminate the +subjects (because you dialed in the power, ISO and aperture before and haven't +changed them) and the ambient fill in and add movement (because you adjusted +the shutter speed). + +Your first few shots will probably be underexposed or overexposed and have too +much or too little blur. Don't worry, it always takes me at least a half hour +to start taking decent photos this way. You really need to play with the +settings as you go to find out what's going to work for the lighting *that +night*. + +#### Things to Watch Out For + +There are a couple of tricky parts to this style of shooting that I'll +mention. First I'll talk about the typical problems you'll see right away. +More than one of these certainly might apply; fix them one at a time. + + + GW-0656 + +* **If the background is dark or there is not enough blur,** you need to use a + longer shutter speed to let more ambient light in. +* **If the dancers are not sharp enough,** you need *more* flash power. Turn + it up a little bit at a time. +* **If people's shirts, faces and limbs are overexposed *and not blurry*,** + you need to turn down the flash. Bring it down a bit by reducing the power, + narrowing the aperture or reducing the ISO. +* **If shirts, faces and limbs are overexposed *and blurry*,** you need to + shorten the shutter speed to let less ambient light in. + +I'm warning you now, fixing one of these will probably fuck something else up. +It can be really infuriating, but if you slow down, stay calm and think +through it logically you'll be able to narrow it down and figure out what you +need. It's a really good feeling when you finally nail it. + +There are two other tricky problems that I'm still working on myself: + +* **Dances are unevenly lit.** When you walk over to the other side of the + room to shoot, you'll have to adjust the shutter speed to compensate for the + different amount of ambient light. You really only have a range of about + half a stop where the ambient gives you the right amount of blur, so you + have to be careful and stay on your toes. +* **Watch where you bounce.** In rooms with low ceilings, where you point the + flash on the ceiling has a huge effect on the lighting. You need to be + conscious of what part of the ceiling you're bouncing from to get good + results. + +Good Luck! +---------- + +I hope this post helped you out. Even if you hate how my photos look and don't +want to make anything like them, at least it will show you what *not* to do. +I'd love to hear comments or advice on what you like or don't like, and what I +could do better. If you've got questions I'll do my best to answer them too! + +Now go dance, have fun, and make beautiful photographs so we can all remember +the fantastic events we make happen! + + + GW-0438 + + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2009/03/candy-colored-terminal.html --- a/content/blog/2009/03/candy-colored-terminal.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,97 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "Candy Colored Terminal" - snip: "Better colors for the OS X Terminal." - created: 2009-03-18 18:26:28 -%} - -{% block article %} - -Yesterday I wrote a blog post about [adding Mercurial information to your bash -prompt](/blog/entry/2009/3/17/mercurial-bash-prompts/). Almost all of the -comments on that entry so far have been asking for the colors I used in the -Terminal screenshots, so that's what this post is for. - -The first section of this entry will be able overcoming the limitations of the -OS X Terminal application. Skip it if you're using another (better) terminal -application. - -The OS X Terminal and Colors ----------------------------- - -OS X's Terminal doesn't support 256-color mode. I have no idea why. Apple, if -you're reading, *please fix this*. Because of this, if you want to make the -colors prettier you've only got the 16 ANSI colors to work with. - -Once again, Apple fails hard with Terminal.app. You can't change the 16 ANSI -colors -- you can only change the default text color and background. -Seriously, Apple, what the hell? - -To fix this, you'll need to install [SIMBL][] and [TerminalColors][]. Follow -the instructions on those pages and be sure to restart Terminal once you've -finished. Now you can change the ANSI colors be hitting the "More" button -under the text color settings. - -[SIMBL]: http://www.culater.net/software/SIMBL/SIMBL.php -[TerminalColors]: http://ciaranwal.sh/2007/11/01/customising-colours-in-leopard-terminal - -Picking Some Colors -------------------- - -Now that we've beaten Terminal.app into compliance, it's time to pick some -pretty colors. I love the [Monokai][] color scheme for [TextMate][] and so I -based my choices on that. Here's what it looks like: - -![Screenshot of my Terminal, with colors.](/media/images{{parent_url}}/terminal-colors.png "My Terminal colors.") - -[Monokai]: http://www.monokai.nl/blog/2006/07/15/textmate-color-theme/ -[TextMate]: http://macromates.com/ - -Here are the RGB values I'm using: - -* Black: 0, 0, 0 -* Red: 229, 34, 34 -* Green: 166, 227, 45 -* Yellow: 252, 149, 30 -* Blue: 196, 141, 255 -* Magenta: 250, 37, 115 -* Cyan: 103, 217, 240 -* White: 242, 242, 242 - -The window background is just set to plain old 0, 0, 0 with a 95% opacity. - -Not all of the colors match the names: "blue" is actually more of a purple, -and "yellow" is much more orange. That doesn't really matter much though -because they're different enough from each other to still be useful. - -Now that the colors are set up you can use them in the normal way. Check out -[this guide][] if you don't know how to do that. Here's the full code for my -bash prompt: - -**UPDATE:** I've cleaned up the code a lot thanks to the kind folks in #bash -on freenode who helped set me straight on bash quoting and escaping. I've also -switched to using my [hg-prompt extension](/projects/hg-prompt/) for the -repository information. - -[this guide]: http://www.ibm.com/developerworks/linux/library/l-tip-prompt/ - - :::bash - D=$'\e[37;40m' - PINK=$'\e[35;40m' - GREEN=$'\e[32;40m' - ORANGE=$'\e[33;40m' - - hg_ps1() { - hg prompt "{${D} on ${PINK}{branch}}{${D} at ${ORANGE}{bookmark}}{${GREEN}{status}}" 2> /dev/null - } - - export PS1='\n${PINK}\u ${D}at ${ORANGE}\h ${D}in ${GREEN}\w$(hg_ps1)\ - ${D}\n$ ' - -I hope this helps! If you'd like to share your color schemes or tell me how to -make the code for my prompt a bit less ugly, please find me on [Twitter][twsl]! - -[twsl]: {{links.twsl}} - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2009/03/candy-colored-terminal.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2009/03/candy-colored-terminal.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,95 @@ ++++ +title = "Candy Colored Terminal" +snip = "Better colors for the OS X Terminal." +date = 2009-03-18T18:26:28Z +draft = false + ++++ + +Yesterday I wrote a blog post about [adding Mercurial information to your bash +prompt](/blog/entry/2009/3/17/mercurial-bash-prompts/). Almost all of the +comments on that entry so far have been asking for the colors I used in the +Terminal screenshots, so that's what this post is for. + +The first section of this entry will be able overcoming the limitations of the +OS X Terminal application. Skip it if you're using another (better) terminal +application. + +The OS X Terminal and Colors +---------------------------- + +OS X's Terminal doesn't support 256-color mode. I have no idea why. Apple, if +you're reading, *please fix this*. Because of this, if you want to make the +colors prettier you've only got the 16 ANSI colors to work with. + +Once again, Apple fails hard with Terminal.app. You can't change the 16 ANSI +colors — you can only change the default text color and background. +Seriously, Apple, what the hell? + +To fix this, you'll need to install [SIMBL][] and [TerminalColors][]. Follow +the instructions on those pages and be sure to restart Terminal once you've +finished. Now you can change the ANSI colors be hitting the "More" button +under the text color settings. + +[SIMBL]: http://www.culater.net/software/SIMBL/SIMBL.php +[TerminalColors]: http://ciaranwal.sh/2007/11/01/customising-colours-in-leopard-terminal + +Picking Some Colors +------------------- + +Now that we've beaten Terminal.app into compliance, it's time to pick some +pretty colors. I love the [Monokai][] color scheme for [TextMate][] and so I +based my choices on that. Here's what it looks like: + +![Screenshot of my Terminal, with colors.](/media/images/blog/2009/03/terminal-colors.png "My Terminal colors.") + +[Monokai]: http://www.monokai.nl/blog/2006/07/15/textmate-color-theme/ +[TextMate]: http://macromates.com/ + +Here are the RGB values I'm using: + +* Black: 0, 0, 0 +* Red: 229, 34, 34 +* Green: 166, 227, 45 +* Yellow: 252, 149, 30 +* Blue: 196, 141, 255 +* Magenta: 250, 37, 115 +* Cyan: 103, 217, 240 +* White: 242, 242, 242 + +The window background is just set to plain old 0, 0, 0 with a 95% opacity. + +Not all of the colors match the names: "blue" is actually more of a purple, +and "yellow" is much more orange. That doesn't really matter much though +because they're different enough from each other to still be useful. + +Now that the colors are set up you can use them in the normal way. Check out +[this guide][] if you don't know how to do that. Here's the full code for my +bash prompt: + +**UPDATE:** I've cleaned up the code a lot thanks to the kind folks in #bash +on freenode who helped set me straight on bash quoting and escaping. I've also +switched to using my [hg-prompt extension](/projects/hg-prompt/) for the +repository information. + +[this guide]: http://www.ibm.com/developerworks/linux/library/l-tip-prompt/ + +```bash +D=$'\e[37;40m' +PINK=$'\e[35;40m' +GREEN=$'\e[32;40m' +ORANGE=$'\e[33;40m' + +hg_ps1() { + hg prompt "{${D} on ${PINK}{branch}}{${D} at ${ORANGE}{bookmark}}{${GREEN}{status}}" 2> /dev/null +} + +export PS1='\n${PINK}\u ${D}at ${ORANGE}\h ${D}in ${GREEN}\w$(hg_ps1)\ +${D}\n$ ' +``` + +I hope this helps! If you'd like to share your color schemes or tell me how to +make the code for my prompt a bit less ugly, please find me on [Twitter][twsl]! + +[twsl]: {{links.twsl}} + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2009/03/mercurial-bash-prompts.html --- a/content/blog/2009/03/mercurial-bash-prompts.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,184 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "Mercurial Bash Prompts" - snip: "Always know where you are." - created: 2009-03-17 21:34:55 -%} - -{% block article %} - -I've been spending a lot of time in the Terminal lately. I use bash, and it -lets you configure the prompt pretty much however you want. I won't go into -how to do the most basic configuration here - if you want to get up to speed -check out this [guide][]. - -In this post I'm just going to talk about one simple addition that I -personally find very helpful: displaying the current branch of a [Mercurial][] -repository. - -Update: the hg-prompt Extension ---------------- - -I swear, this is the last time I'm updating this entry. I went ahead and made -an extension for Mercurial called -[hg-prompt](http://stevelosh.com/projects/hg-prompt/) that does everything -much more elegantly. **Use that instead!** - -[guide]: http://www.ibm.com/developerworks/linux/library/l-tip-prompt/ -[Mercurial]: {{links.mercurial}} -[git version]: http://gist.github.com/31631 - -My Starting Point ------------------ - -Here's what my prompt looked like a couple of days ago: - -![My bash prompt without the branch displayed](/media/images{{parent_url}}/prompt-without-branch.png "My bash prompt without the branch displayed.") - -Here's the code in my `.bashrc` file to create it. I've stripped out the color -information to save space. - - :::bash - export PS1='\n\u at \h in \w\n$ ' - -I use the same prompt on every computer I work with, so with this prompt I can -see which user I'm logged in as, which machine I'm on, and where in the -filesystem I am. It's a lot of useful information at a glance but doesn't seem -too cluttered. - -I have a linebreak in there because the filesystem path can get pretty long. -If I kept it all on one line most of my commands would be wrapped around -anyway, which I find harder to read. - -Adding the Mercurial Branch ---------------------------- - -I use Mercurial for version control, and I use branches quite a bit when I'm -developing. One problem with this is that it's easy to forget which branch I'm -on at a given moment. I *could* just use `hg branch` to find out, but that -gets tedious to type, even when aliased to something like `hb`. - -A few days ago I got the idea that I could just display the current branch on -my prompt whenever I'm in a directory that's part of a repository. Here's what -my prompt looks like now: - -![My bash prompt with the branch displayed](/media/images{{parent_url}}/prompt-with-branch.png "My bash prompt with the branch displayed.") - -And here's the code in my `.bashrc` that does it: - - :::bash - hg_in_repo() { - hg branch 2> /dev/null | awk '{print "on "}' - } - - hg_branch() { - hg branch 2> /dev/null | awk '{print $1}' - } - - export PS1='\n\u at \h in \w $(hg_in_repo)$(hg_branch)\n$ ' - -The `on branchname` piece is only displayed when you're in a directory that's -part of a Mercurial repository. - -I've split it up into two separate functions because I wanted to have `on` and -`branchname` displayed in two different colors. I couldn't seem to include the -color codes in the awk command, so I split it up and put the colors in the -export statement with the rest of them. If you don't care about colors (or -don't mind having both words the same color) you can just collapse it into one -function. - -Updated: Is It Dirty? ---------------------- - -After I posted this entry Matt Kemp commented with a link to a [git -version][]. One feature that version has is a simple indicator of whether or -not the repository you're in is dirty. I ported it to Mercurial and here's the -result: - -![My bash prompt with the branch and dirty indicator displayed](/media/images{{parent_url}}/prompt-with-dirty.png "My bash prompt with the branch and dirty indicator displayed.") - -And the code in `.bashrc`: - - :::bash - hg_dirty() { - hg status --no-color 2> /dev/null \ - | awk '$1 == "?" { print "?" } $1 != "?" { print "!" }' \ - | sort | uniq | head -c1 - } - - hg_in_repo() { - [[ `hg branch 2> /dev/null` ]] && echo 'on ' - } - - hg_branch() { - hg branch 2> /dev/null - } - - export PS1='\n\u at \h in \w $(hg_in_repo)$(hg_branch)$(hg_dirty)\n$ ' - -This gives you a `?` after the branch when there are untracked files (and -*only* untracked files), and a `!` if there are any modified, tracked files. - -Updated: Bookmarks Too! ----------------- - -I've added another piece to show bookmarks as well. I've also figured out how -to add colors directly in the functions, so here's the (much nicer) updated -code all at once: - - :::bash - DEFAULT="[37;40m" - PINK="[35;40m" - GREEN="[32;40m" - ORANGE="[33;40m" - - hg_dirty() { - hg status --no-color 2> /dev/null \ - | awk '$1 == "?" { unknown = 1 } - $1 != "?" { changed = 1 } - END { - if (changed) printf "!" - else if (unknown) printf "?" - }' - } - - hg_branch() { - hg branch 2> /dev/null | \ - awk '{ printf "\033[37;0m on \033[35;40m" $1 }' - hg bookmarks 2> /dev/null | \ - awk '/\*/ { printf "\033[37;0m at \033[33;40m" $2 }' - } - - export PS1='\n\e${PINK}\u \ - \e${DEFAULT}at \e${ORANGE}\h \ - \e${DEFAULT}in \e${GREEN}\w\ - $(hg_branch)\e${GREEN}$(hg_dirty)\ - \e${DEFAULT}\n$ ' - -These are some pretty simple changes but they help keep me sane. One thing to -be aware of: if you use all of these it does slow down the rendering of the -prompt by a tiny, but noticeable, amount. I'm not the strongest bash scripter, -so if there's a better way to do this (or a way that will make it faster and -reduce the delay) please let me know! - -**UPDATE:** Matt Kemp posted a link to a [git version][] of this below. If you -use git, check it out! One thing that version has that I didn't think of is an -indicator of whether the repository is dirty (has uncommitted changes). I'm -going to go ahead and steal that idea for my prompt too. - -**UPDATE:** By request, I've written [an entry about the -colors](/blog/entry/2009/3/18/candy-colored-terminal/). - -**UPDATE:** Kevin Bullock pointed out that the Python interpreter needed to be -started a bunch of times which will degrade performance. I've changed up the -"dirty" code a bit to reduce the number of interpreters needed. It's still not -as efficient as his version, but I think it's about as good as I'm going to -get if I want separate colors for the pieces and don't want to rely on an -external script. - -**FINAL UPDATE:** I made an extension for Mercurial called -[hg-prompt](http://stevelosh.com/projects/hg-prompt/) that does everything -much more elegantly. **Use that instead!** - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2009/03/mercurial-bash-prompts.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2009/03/mercurial-bash-prompts.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,185 @@ ++++ +title = "Mercurial Bash Prompts" +snip = "Always know where you are." +date = 2009-03-17T21:34:55Z +draft = false + ++++ + +I've been spending a lot of time in the Terminal lately. I use bash, and it +lets you configure the prompt pretty much however you want. I won't go into +how to do the most basic configuration here - if you want to get up to speed +check out this [guide][]. + +In this post I'm just going to talk about one simple addition that I +personally find very helpful: displaying the current branch of a [Mercurial][] +repository. + +Update: the hg-prompt Extension +--------------- + +I swear, this is the last time I'm updating this entry. I went ahead and made +an extension for Mercurial called +[hg-prompt](http://stevelosh.com/projects/hg-prompt/) that does everything +much more elegantly. **Use that instead!** + +[guide]: http://www.ibm.com/developerworks/linux/library/l-tip-prompt/ +[Mercurial]: {{links.mercurial}} +[git version]: http://gist.github.com/31631 + +My Starting Point +----------------- + +Here's what my prompt looked like a couple of days ago: + +![My bash prompt without the branch displayed](/media/images/blog/2009/03/prompt-without-branch.png "My bash prompt without the branch displayed.") + +Here's the code in my `.bashrc` file to create it. I've stripped out the color +information to save space. + +```bash +export PS1='\n\u at \h in \w\n$ ' +``` + +I use the same prompt on every computer I work with, so with this prompt I can +see which user I'm logged in as, which machine I'm on, and where in the +filesystem I am. It's a lot of useful information at a glance but doesn't seem +too cluttered. + +I have a linebreak in there because the filesystem path can get pretty long. +If I kept it all on one line most of my commands would be wrapped around +anyway, which I find harder to read. + +Adding the Mercurial Branch +--------------------------- + +I use Mercurial for version control, and I use branches quite a bit when I'm +developing. One problem with this is that it's easy to forget which branch I'm +on at a given moment. I *could* just use `hg branch` to find out, but that +gets tedious to type, even when aliased to something like `hb`. + +A few days ago I got the idea that I could just display the current branch on +my prompt whenever I'm in a directory that's part of a repository. Here's what +my prompt looks like now: + +![My bash prompt with the branch displayed](/media/images/blog/2009/03/prompt-with-branch.png "My bash prompt with the branch displayed.") + +And here's the code in my `.bashrc` that does it: + +```bash +hg_in_repo() { + hg branch 2> /dev/null | awk '{print "on "}' +} + +hg_branch() { + hg branch 2> /dev/null | awk '{print $1}' +} + +export PS1='\n\u at \h in \w $(hg_in_repo)$(hg_branch)\n$ ' +``` + +The `on branchname` piece is only displayed when you're in a directory that's +part of a Mercurial repository. + +I've split it up into two separate functions because I wanted to have `on` and +`branchname` displayed in two different colors. I couldn't seem to include the +color codes in the awk command, so I split it up and put the colors in the +export statement with the rest of them. If you don't care about colors (or +don't mind having both words the same color) you can just collapse it into one +function. + +Updated: Is It Dirty? +--------------------- + +After I posted this entry Matt Kemp commented with a link to a [git +version][]. One feature that version has is a simple indicator of whether or +not the repository you're in is dirty. I ported it to Mercurial and here's the +result: + +![My bash prompt with the branch and dirty indicator displayed](/media/images/blog/2009/03/prompt-with-dirty.png "My bash prompt with the branch and dirty indicator displayed.") + +And the code in `.bashrc`: + +```bash +hg_dirty() { + hg status --no-color 2> /dev/null \ + | awk '$1 == "?" { print "?" } $1 != "?" { print "!" }' \ + | sort | uniq | head -c1 +} + +hg_in_repo() { + [[ `hg branch 2> /dev/null` ]] && echo 'on ' +} + +hg_branch() { + hg branch 2> /dev/null +} + +export PS1='\n\u at \h in \w $(hg_in_repo)$(hg_branch)$(hg_dirty)\n$ ' +``` + +This gives you a `?` after the branch when there are untracked files (and +*only* untracked files), and a `!` if there are any modified, tracked files. + +Updated: Bookmarks Too! +---------------- + +I've added another piece to show bookmarks as well. I've also figured out how +to add colors directly in the functions, so here's the (much nicer) updated +code all at once: + +```bash +DEFAULT="[37;40m" +PINK="[35;40m" +GREEN="[32;40m" +ORANGE="[33;40m" + +hg_dirty() { + hg status --no-color 2> /dev/null \ + | awk '$1 == "?" { unknown = 1 } + $1 != "?" { changed = 1 } + END { + if (changed) printf "!" + else if (unknown) printf "?" + }' +} + +hg_branch() { + hg branch 2> /dev/null | \ + awk '{ printf "\033[37;0m on \033[35;40m" $1 }' + hg bookmarks 2> /dev/null | \ + awk '/\*/ { printf "\033[37;0m at \033[33;40m" $2 }' +} + +export PS1='\n\e${PINK}\u \ +\e${DEFAULT}at \e${ORANGE}\h \ +\e${DEFAULT}in \e${GREEN}\w\ +$(hg_branch)\e${GREEN}$(hg_dirty)\ +\e${DEFAULT}\n$ ' +``` + +These are some pretty simple changes but they help keep me sane. One thing to +be aware of: if you use all of these it does slow down the rendering of the +prompt by a tiny, but noticeable, amount. I'm not the strongest bash scripter, +so if there's a better way to do this (or a way that will make it faster and +reduce the delay) please let me know! + +**UPDATE:** Matt Kemp posted a link to a [git version][] of this below. If you +use git, check it out! One thing that version has that I didn't think of is an +indicator of whether the repository is dirty (has uncommitted changes). I'm +going to go ahead and steal that idea for my prompt too. + +**UPDATE:** By request, I've written [an entry about the +colors](/blog/entry/2009/3/18/candy-colored-terminal/). + +**UPDATE:** Kevin Bullock pointed out that the Python interpreter needed to be +started a bunch of times which will degrade performance. I've changed up the +"dirty" code a bit to reduce the number of interpreters needed. It's still not +as efficient as his version, but I think it's about as good as I'm going to +get if I want separate colors for the pieces and don't want to rely on an +external script. + +**FINAL UPDATE:** I made an extension for Mercurial called +[hg-prompt](http://stevelosh.com/projects/hg-prompt/) that does everything +much more elegantly. **Use that instead!** + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2009/04/why-people-dont-like-metal.html --- a/content/blog/2009/04/why-people-dont-like-metal.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,130 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "Why People Don’t Like Metal" - snip: "It’s probably not what you think." - created: 2009-04-02 22:32:27 -%} - -{% block article %} - -I don't think I've ever written a blog entry strictly about music. Usually -I'll reference it when I write about dancing, but today I just want to talk -about music. In particular, metal. - -"Metal" can refer to a lot of music. For this post I'm referring to the newer -forms of metal that sprung up in the past decade or so. One example of this is -[Should Have Stayed in the -Shallows](http://www.youtube.com/watch?v=hmkLQRp8htY) by [Fear Before (the -March of Flames)](http://www.fearbefore.net/). I know there are many, many -more bands out there but this one song is a good sample of what I want to talk -about. - -"I Don't Like all the Screaming" ------- - -If you play a metal song for someone that doesn't usually listen to the genre, -this is probably the first thing you'll hear. There might be a mention of -"everything is so loud" but almost without fail the screaming is what seems to -turn people off. Why is that? - -The vocals of a song are usually the most noticeable part, especially for -nonmusicians. Most people who don't play an instrument don't have the -background to absorb and appreciate complicated instrumental work in any kind -of music, but *everyone has vocal cords*. Everyone can appreciate speech and -singing at some level. - -So what is it about screaming that most people don't like? If you ask them, -you'll get an answer, but I don't think it's the correct one. - -"I Can't Understand the Words" ------ - -This is what you'll usually hear when you press the issue. It seems reasonable --- the lyrics to most metal bands are indecipherable without a written copy, -even to people that love them. Surely this is the problem? - -I don't think it is. To get at the *real* issue, I usually respond with one -more question: - -**Do you enjoy any bands whose vocalist sings in a language you don't know?** - -I've never had anyone tell me: "I can't stand *any* music not in my own -language." I'm sure there are some out there that feel that way, but I think -they would be in the minority. I've met plenty of people that enjoy hearing -[Sigur Rós](http://www.sigur-ros.co.uk/) and I'm fairly certain that *exactly -none of them* speak Icelandic fluently. Many don't even pronounce the name -correctly. - -If people enjoy listening to music with vocals in a language they don't -understand, then their dislike of metal must not stem purely from -unintelligible lyrics. What could it come from? - -Timbre ------- - -Here's one idea: many people are put off by the -[timbre](http://en.wikipedia.org/wiki/Timbre) of a metal band. - -Timbre usually refers to the quality of a musical sound or note. It's what -makes an E on a guitar sound different than an E on a cello, even if it's the -same pitch. It's how you can tell your mother's voice from your friend's -voice, even if she speaks at the same frequency. - -A concept I first came across when reading [This Is Your Brain On -Music](http://www.amazon.com/This-Your-Brain-Music-Obsession/dp/0452288525/) -is "the timbre of a band as a whole." The idea of timbre can be expanded to -neatly describe why two bands playing the same song in the same key can sound -*completely* different. The instruments of the band each contribute their own -sound and when taken together you have a timbre just as unique as a particular -instrument's. - -Metal bands each have their own timbre, but they're all related -- that's what -makes them into a genre. A screaming vocalist is an extremely distinct element -of this that isn't really found anywhere else. Perhaps the answer to "why do -people dislike metal?" is that they simply don't enjoy the timbre, just as -some people don't enjoy violins or saxophones. - -Lack of Musical Background ------ - -This is another possible answer. As I mentioned before, everyone has vocal -cords. Everyone can relate to a singer through the words they're singing. Even -if the words are in a different language they can still relate to the act of -singing. Not everyone knows how to sing well, but I believe you'd be hard -pressed to find someone that has never sung in the shower or somewhere just as -private. - -Very few people can (or want to) relate to screaming. Screaming is something -we usually do only when threatened or angry, which is hopefully a minority of -our lives. So once the vocalist is screaming constantly, people that don't -enjoy metal no longer have the element they're most used to focusing on. - -What's left? If you can't understand the words, the voice becomes another -instrument instead of something "special." Many people simply haven't ever -tried listening to and appreciating purely instrumental music and so they lose -interest. - -What Can We Do? ------ - -I'm sure I haven't completely nailed down the reasons why people don't enjoy -metal. It's almost certainly a combination of quite a few things. But if -someone is genuinely interested in learning more about metal and about why -people like it -- perhaps a close friend listens and they'd like to know more --- I think there's two things they can do that may help. - -**Think of the vocals as another instrument, and listen to how they interact -with the rest of the band.** - -Admittedly, there's a lot of really terrible metal out there. As the genre has -expanded in popularity a lot of "musicians" (barely) have jumped in and -created bands no more musically meaningful than the average pop singer. It's -easy to come across this and dismiss the whole genre as a bunch of untalented -hacks. - -Word of mouth is probably the best way to pick out the good from the bad. If -you look hard enough you can find some amazing music that you might grow to -love and appreciate. - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2009/04/why-people-dont-like-metal.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2009/04/why-people-dont-like-metal.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,127 @@ ++++ +title = "Why People Don’t Like Metal" +snip = "It’s probably not what you think." +date = 2009-04-02T22:32:27Z +draft = false + ++++ + +I don't think I've ever written a blog entry strictly about music. Usually +I'll reference it when I write about dancing, but today I just want to talk +about music. In particular, metal. + +"Metal" can refer to a lot of music. For this post I'm referring to the newer +forms of metal that sprung up in the past decade or so. One example of this is +[Should Have Stayed in the +Shallows](http://www.youtube.com/watch?v=hmkLQRp8htY) by [Fear Before (the +March of Flames)](http://www.fearbefore.net/). I know there are many, many +more bands out there but this one song is a good sample of what I want to talk +about. + +"I Don't Like all the Screaming" +------ + +If you play a metal song for someone that doesn't usually listen to the genre, +this is probably the first thing you'll hear. There might be a mention of +"everything is so loud" but almost without fail the screaming is what seems to +turn people off. Why is that? + +The vocals of a song are usually the most noticeable part, especially for +nonmusicians. Most people who don't play an instrument don't have the +background to absorb and appreciate complicated instrumental work in any kind +of music, but *everyone has vocal cords*. Everyone can appreciate speech and +singing at some level. + +So what is it about screaming that most people don't like? If you ask them, +you'll get an answer, but I don't think it's the correct one. + +"I Can't Understand the Words" +----- + +This is what you'll usually hear when you press the issue. It seems reasonable +-- the lyrics to most metal bands are indecipherable without a written copy, +even to people that love them. Surely this is the problem? + +I don't think it is. To get at the *real* issue, I usually respond with one +more question: + +**Do you enjoy any bands whose vocalist sings in a language you don't know?** + +I've never had anyone tell me: "I can't stand *any* music not in my own +language." I'm sure there are some out there that feel that way, but I think +they would be in the minority. I've met plenty of people that enjoy hearing +[Sigur Rós](http://www.sigur-ros.co.uk/) and I'm fairly certain that *exactly +none of them* speak Icelandic fluently. Many don't even pronounce the name +correctly. + +If people enjoy listening to music with vocals in a language they don't +understand, then their dislike of metal must not stem purely from +unintelligible lyrics. What could it come from? + +Timbre +------ + +Here's one idea: many people are put off by the +[timbre](http://en.wikipedia.org/wiki/Timbre) of a metal band. + +Timbre usually refers to the quality of a musical sound or note. It's what +makes an E on a guitar sound different than an E on a cello, even if it's the +same pitch. It's how you can tell your mother's voice from your friend's +voice, even if she speaks at the same frequency. + +A concept I first came across when reading [This Is Your Brain On +Music](http://www.amazon.com/This-Your-Brain-Music-Obsession/dp/0452288525/) +is "the timbre of a band as a whole." The idea of timbre can be expanded to +neatly describe why two bands playing the same song in the same key can sound +*completely* different. The instruments of the band each contribute their own +sound and when taken together you have a timbre just as unique as a particular +instrument's. + +Metal bands each have their own timbre, but they're all related — that's what +makes them into a genre. A screaming vocalist is an extremely distinct element +of this that isn't really found anywhere else. Perhaps the answer to "why do +people dislike metal?" is that they simply don't enjoy the timbre, just as +some people don't enjoy violins or saxophones. + +Lack of Musical Background +----- + +This is another possible answer. As I mentioned before, everyone has vocal +cords. Everyone can relate to a singer through the words they're singing. Even +if the words are in a different language they can still relate to the act of +singing. Not everyone knows how to sing well, but I believe you'd be hard +pressed to find someone that has never sung in the shower or somewhere just as +private. + +Very few people can (or want to) relate to screaming. Screaming is something +we usually do only when threatened or angry, which is hopefully a minority of +our lives. So once the vocalist is screaming constantly, people that don't +enjoy metal no longer have the element they're most used to focusing on. + +What's left? If you can't understand the words, the voice becomes another +instrument instead of something "special." Many people simply haven't ever +tried listening to and appreciating purely instrumental music and so they lose +interest. + +What Can We Do? +----- + +I'm sure I haven't completely nailed down the reasons why people don't enjoy +metal. It's almost certainly a combination of quite a few things. But if +someone is genuinely interested in learning more about metal and about why +people like it — perhaps a close friend listens and they'd like to know more +— I think there's two things they can do that may help. + +**Think of the vocals as another instrument, and listen to how they interact +with the rest of the band.** + +Admittedly, there's a lot of really terrible metal out there. As the genre has +expanded in popularity a lot of "musicians" (barely) have jumped in and +created bands no more musically meaningful than the average pop singer. It's +easy to come across this and dismiss the whole genre as a bunch of untalented +hacks. + +Word of mouth is probably the best way to pick out the good from the bad. If +you look hard enough you can find some amazing music that you might grow to +love and appreciate. + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2009/05/what-i-hate-about-mercurial.html --- a/content/blog/2009/05/what-i-hate-about-mercurial.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,232 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "What I Hate About Mercurial" - snip: "Hg, I love you, but sometimes you bring me down." - created: 2009-05-29 19:51:05 -%} - -{% block article %} - -This entry was inspired by [Jacob Kaplan-Moss][JKM], who was inspired by -[Titus][], who was inspired by [brian d foy][BDF]. The premise is that you -can't really claim to know much about a piece of software until you can name a -few things you hate about it. - -[JKM]: http://jacobian.org/writing/hate-python/ -[Titus]: http://ivory.idyll.org/blog/mar-07/five-things-I-hate-about-python -[BDF]: http://use.perl.org/~brian_d_foy/journal/32556?from=rss - -Jacob and Titus talked about Python, and so have a bunch of other people, so I -figured I'd write about something a bit different: -[Mercurial](http://selenic.com/mercurial). - -I love Mercurial to death, but there *are* a few things about it that annoy -me. Here they are, in no particular order. - -[TOC] - -Configuration Through a Textfile --------------------------------- - -This is one of the things that [git](http://git-scm.com/) people seem to like -to [bring up](http://jointheconversation.org/2008/11/24/on-mercurial.html): -"It doesn't have a command to set your username and email and such? Lame." - -I personally don't mind editing a text file -- `vim ~/.hgrc` really isn't that -hard to type and the format is simple enough that you get the hang of it in -about ten seconds. It forces you to know *where* the config file is, which is -nice when the magic "Oh man, I could put my config files under *version -control*!" moment strikes. - -That said, this is on the list because I'd like to see a command to edit some -of the common options just so people will *stop complaining* about something -so trivial. - -"hg rm" is a Confusing Mess ---------------------------- - -Here's part of the help for the `hg rm` command: - -> This only removes files from the current branch, not from the entire -> project history. -A can be used to remove only files that have already -> been deleted, -f can be used to force deletion, and -Af can be used -> to remove files from the next revision without deleting them. - -What the hell? If `-A` won't remove files that are still present, and `-f` -forces the files to be deleted, why the fuck does combining them mean *the -exact opposite of both*? - -I had to look up the syntax every single time I wanted to use this command, -until I added this alias to my `~/.hgrc`: - - :::ini - [alias] - untrack = rm -Af - -Now I can use `hg untrack whatever.py` to stop tracking a file. - -Addremove Can Track Renames But Won't Unless You Ask It Really Nicely ---------------------------------------------------------------------- - -It took me a while to realize this, but Mercurial can actually record file -renames -- not just separate adds and removes -- when using `hg addremove`. -Here's what happens when you move a file and then use `hg addremove` normally -to have Mercurial track the changes. - - :::console - sjl at ecgtheow in ~/Desktop/test on default - $ ls - total 16 - -rw-r--r-- 1 sjl 14B May 29 20:12 a - -rw-r--r-- 1 sjl 12B May 29 20:12 b - - sjl at ecgtheow in ~/Desktop/test on default - $ mv b c - - sjl at ecgtheow in ~/Desktop/test on default! - $ hg addremove - removing b - adding c - - sjl at ecgtheow in ~/Desktop/test on default! - $ hg diff - diff --git a/b b/b - deleted file mode 100644 - --- a/b - +++ /dev/null - @@ -1,1 +0,0 @@ - -To you too! - diff --git a/c b/c - new file mode 100644 - --- /dev/null - +++ b/c - @@ -0,0 +1,1 @@ - +To you too! - - sjl at ecgtheow in ~/Desktop/test on default! - $ hg commit -m 'Normal addremove.' - - sjl at ecgtheow in ~/Desktop/test on default - $ - -Now watch what happens when we tell Mercurial to detect renames when using `hg -addremove`: - - :::console - sjl at ecgtheow in ~/Desktop/test on default - $ ls - total 16 - -rw-r--r-- 1 sjl 14B May 29 20:12 a - -rw-r--r-- 1 sjl 12B May 29 20:12 c - - sjl at ecgtheow in ~/Desktop/test on default - $ mv c b - - sjl at ecgtheow in ~/Desktop/test on default! - $ hg addremove --similarity 100 - adding b - removing c - recording removal of c as rename to b (100% similar) - - sjl at ecgtheow in ~/Desktop/test on default! - $ hg diff - diff --git a/c b/b - rename from c - rename to b - - sjl at ecgtheow in ~/Desktop/test on default! - $ hg commit -m 'This time with rename detection.' - - sjl at ecgtheow in ~/Desktop/test on default - $ - -This time it notices the rename, and records it. The diff is far, far easier -to read, and if a branch is merged in where someone else changed that file, -the changes will follow it over. - -The `--similarity` option is a percentage. I have an entry in my `~/.hgrc` -file to default that to 100, which means that renames will only be -automatically detected if the files are *identical*. It's safer, but might not -always catch everything. - -I wish I had known this earlier or that Mercurial defaulted to 100% to catch -the obvious renames. - -And yes, I realize I could use `hg rename` to rename it and it *would* get -recorded, but usually I'm moving files by some other method and using `hg -addremove` to clean up later. - -Now, while we're on the topic... - -Why the Hell Does Status Not Show Renames? ------------------------------------------- - -Assuming they're recorded, I wish `hg status` would show that a file has been -renamed. Here's what we get instead: - - :::console - sjl at ecgtheow in ~/Desktop/test on default - $ hg rename b c - - sjl at ecgtheow in ~/Desktop/test on default! - $ hg stat - A c - R b - - sjl at ecgtheow in ~/Desktop/test on default! - $ hg diff - diff --git a/b b/c - rename from b - rename to c - -No, `hg status`, it clearly *wasn't* an add and a remove, according to `hg -diff`. Why doesn't `hg status` have a separate character for "renamed" files -so we can tell them apart? - -Get Git Out of My Mercurial ---------------------------- - -I *hate* that half of Mercurial's commands need a `--git` option to be really -useful. Is there any reason not to make this the default format and have a -`--useless` option for backwards compatibility? - -I always add it to the defaults in my `~/.hgrc` but it makes me feel kind of -dirty when I do. It adds a bunch of unnecessary lines to the config file and -confuses new people. - -BitBucket Could Be Prettier ---------------------------- - -Don't get me wrong, [BitBucket](http://bitbucket.org/) is an awesome site, but -compared to [GitHub](http://github.com/) it looks a bit dull and unappealing. - -It's definitely more usable (how the hell do I view a graph of all -branches/merges of a given repository on GitHub?) but there's something about -GitHub's design that just makes it pop. - -Mercurial's Site Could Be *Much* Prettier ------------------------------------------ - -Mercurial's site is ugly. Very ugly. It seems strange to me that the ugly -version control system (git) has a fairly good-looking site while the much -more elegant Mercurial has something that looks so boring and dated. - -I know, [mercurial-scm.org](http://mercurial-scm.org/) aims to fix this. Thank -you from the bottom of my heart but please, hurry. The wiki is hurting my -eyes. - -But Hey, at Least It's Not Git! --------------------------------- - -All of those things annoy me, but they're small problems compared to the -revulsion I get when I try to use git every so often. - -Maybe that would be a good topic for another entry. At the very least it'll -probably get me a ton of pageviews and comments saying: "Git's changed, man! -It's not like it used to be, it's totally intuitive now! You just gotta learn -how it stores the data!" - -I learned, and I'll still take Mercurial despite the small annoyances. - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2009/05/what-i-hate-about-mercurial.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2009/05/what-i-hate-about-mercurial.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,233 @@ ++++ +title = "What I Hate About Mercurial" +snip = "Hg, I love you, but sometimes you bring me down." +date = 2009-05-29T19:51:05Z +draft = false + ++++ + +This entry was inspired by [Jacob Kaplan-Moss][JKM], who was inspired by +[Titus][], who was inspired by [brian d foy][BDF]. The premise is that you +can't really claim to know much about a piece of software until you can name a +few things you hate about it. + +[JKM]: http://jacobian.org/writing/hate-python/ +[Titus]: http://ivory.idyll.org/blog/mar-07/five-things-I-hate-about-python +[BDF]: http://use.perl.org/~brian_d_foy/journal/32556?from=rss + +Jacob and Titus talked about Python, and so have a bunch of other people, so I +figured I'd write about something a bit different: +[Mercurial](http://selenic.com/mercurial). + +I love Mercurial to death, but there *are* a few things about it that annoy +me. Here they are, in no particular order. + +{{% toc %}} + +Configuration Through a Textfile +-------------------------------- + +This is one of the things that [git](http://git-scm.com/) people seem to like +to [bring up](http://jointheconversation.org/2008/11/24/on-mercurial.html): +"It doesn't have a command to set your username and email and such? Lame." + +I personally don't mind editing a text file — `vim ~/.hgrc` really isn't that +hard to type and the format is simple enough that you get the hang of it in +about ten seconds. It forces you to know *where* the config file is, which is +nice when the magic "Oh man, I could put my config files under *version +control*!" moment strikes. + +That said, this is on the list because I'd like to see a command to edit some +of the common options just so people will *stop complaining* about something +so trivial. + +"hg rm" is a Confusing Mess +--------------------------- + +Here's part of the help for the `hg rm` command: + +> This only removes files from the current branch, not from the entire +> project history. -A can be used to remove only files that have already +> been deleted, -f can be used to force deletion, and -Af can be used +> to remove files from the next revision without deleting them. + +What the hell? If `-A` won't remove files that are still present, and `-f` +forces the files to be deleted, why the fuck does combining them mean *the +exact opposite of both*? + +I had to look up the syntax every single time I wanted to use this command, +until I added this alias to my `~/.hgrc`: + +```ini +[alias] +untrack = rm -Af +``` + +Now I can use `hg untrack whatever.py` to stop tracking a file. + +Addremove Can Track Renames But Won't Unless You Ask It Really Nicely +--------------------------------------------------------------------- + +It took me a while to realize this, but Mercurial can actually record file +renames — not just separate adds and removes — when using `hg addremove`. +Here's what happens when you move a file and then use `hg addremove` normally +to have Mercurial track the changes. + +```console +sjl at ecgtheow in ~/Desktop/test on default +$ ls +total 16 +-rw-r--r-- 1 sjl 14B May 29 20:12 a +-rw-r--r-- 1 sjl 12B May 29 20:12 b + +sjl at ecgtheow in ~/Desktop/test on default +$ mv b c + +sjl at ecgtheow in ~/Desktop/test on default! +$ hg addremove +removing b +adding c + +sjl at ecgtheow in ~/Desktop/test on default! +$ hg diff +diff --git a/b b/b +deleted file mode 100644 +--- a/b ++++ /dev/null +@@ -1,1 +0,0 @@ +-To you too! +diff --git a/c b/c +new file mode 100644 +--- /dev/null ++++ b/c +@@ -0,0 +1,1 @@ ++To you too! + +sjl at ecgtheow in ~/Desktop/test on default! +$ hg commit -m 'Normal addremove.' + +sjl at ecgtheow in ~/Desktop/test on default +$ +``` + +Now watch what happens when we tell Mercurial to detect renames when using `hg +addremove`: + +```console +sjl at ecgtheow in ~/Desktop/test on default +$ ls +total 16 +-rw-r--r-- 1 sjl 14B May 29 20:12 a +-rw-r--r-- 1 sjl 12B May 29 20:12 c + +sjl at ecgtheow in ~/Desktop/test on default +$ mv c b + +sjl at ecgtheow in ~/Desktop/test on default! +$ hg addremove --similarity 100 +adding b +removing c +recording removal of c as rename to b (100% similar) + +sjl at ecgtheow in ~/Desktop/test on default! +$ hg diff +diff --git a/c b/b +rename from c +rename to b + +sjl at ecgtheow in ~/Desktop/test on default! +$ hg commit -m 'This time with rename detection.' + +sjl at ecgtheow in ~/Desktop/test on default +$ +``` + +This time it notices the rename, and records it. The diff is far, far easier +to read, and if a branch is merged in where someone else changed that file, +the changes will follow it over. + +The `--similarity` option is a percentage. I have an entry in my `~/.hgrc` +file to default that to 100, which means that renames will only be +automatically detected if the files are *identical*. It's safer, but might not +always catch everything. + +I wish I had known this earlier or that Mercurial defaulted to 100% to catch +the obvious renames. + +And yes, I realize I could use `hg rename` to rename it and it *would* get +recorded, but usually I'm moving files by some other method and using `hg +addremove` to clean up later. + +Now, while we're on the topic... + +Why the Hell Does Status Not Show Renames? +------------------------------------------ + +Assuming they're recorded, I wish `hg status` would show that a file has been +renamed. Here's what we get instead: + +```console +sjl at ecgtheow in ~/Desktop/test on default +$ hg rename b c + +sjl at ecgtheow in ~/Desktop/test on default! +$ hg stat +A c +R b + +sjl at ecgtheow in ~/Desktop/test on default! +$ hg diff +diff --git a/b b/c +rename from b +rename to c +``` + +No, `hg status`, it clearly *wasn't* an add and a remove, according to `hg +diff`. Why doesn't `hg status` have a separate character for "renamed" files +so we can tell them apart? + +Get Git Out of My Mercurial +--------------------------- + +I *hate* that half of Mercurial's commands need a `--git` option to be really +useful. Is there any reason not to make this the default format and have a +`--useless` option for backwards compatibility? + +I always add it to the defaults in my `~/.hgrc` but it makes me feel kind of +dirty when I do. It adds a bunch of unnecessary lines to the config file and +confuses new people. + +BitBucket Could Be Prettier +--------------------------- + +Don't get me wrong, [BitBucket](http://bitbucket.org/) is an awesome site, but +compared to [GitHub](http://github.com/) it looks a bit dull and unappealing. + +It's definitely more usable (how the hell do I view a graph of all +branches/merges of a given repository on GitHub?) but there's something about +GitHub's design that just makes it pop. + +Mercurial's Site Could Be *Much* Prettier +----------------------------------------- + +Mercurial's site is ugly. Very ugly. It seems strange to me that the ugly +version control system (git) has a fairly good-looking site while the much +more elegant Mercurial has something that looks so boring and dated. + +I know, [mercurial-scm.org](http://mercurial-scm.org/) aims to fix this. Thank +you from the bottom of my heart but please, hurry. The wiki is hurting my +eyes. + +But Hey, at Least It's Not Git! +-------------------------------- + +All of those things annoy me, but they're small problems compared to the +revulsion I get when I try to use git every so often. + +Maybe that would be a good topic for another entry. At the very least it'll +probably get me a ton of pageviews and comments saying: "Git's changed, man! +It's not like it used to be, it's totally intuitive now! You just gotta learn +how it stores the data!" + +I learned, and I'll still take Mercurial despite the small annoyances. + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2009/06/how-to-contribute-to-mercurial.html --- a/content/blog/2009/06/how-to-contribute-to-mercurial.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,391 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "How to Contribute to Mercurial" - snip: "Ten minutes of setup will make it easier." - created: 2009-06-01 20:09:44 -%} - -{% block article %} - -After my last post on [What I Hate About Mercurial][hate], seydar commented -that I should contribute a patch to add untrack/forget functionality. I -decided to take the advice, stop being lazy and write one. - -It's a very, very simple change and I'm not sure if it will even make it into -the core repository, but even so it gave me a chance to figure out how to set -up a decent development environment for hacking on Mercurial. I figured that -other people might find that information useful so I decided to write this. - -[hate]: http://stevelosh.com/blog/entry/2009/5/29/what-i-hate-about-mercurial/ - -[TOC] - -About This Guide ----------------- - -This is meant to be a guide to get yourself set up to work on Mercurial's code -base and contribute your work back to the community. It's *not* a guide for -actually working on the code. - -I'm going to go ahead and assume a few (small) things: - -1. You're running OS X or some form of Linux. If you're using Windows, I'm - sorry, but I just don't use it enough to really help. -2. You've got [Python][] installed. -3. You've got a copy of Mercurial installed in some way (MacPorts, apt-get, - source, etc). -4. You're familiar with (and have clients for) IRC and email. - -If those things are true, let's get started. The entire process should take -between 15 and 45 minutes, depending on how comfortable you are with bash, -Python, and Mercurial itself. - -[Python]: {{links.python}} - -Join the Community ------------------- - -The first step to contributing is to join the community. Things will probably -go more smoothly if other developers have seen your name before and those -other developers are a great resource if you get stuck. - -Join the [#mercurial channel on Freenode][irc]. A lot of other Mercurial -developers hang out in there and it's a good place to go if you want some -really quick feedback. Take a few minutes and [register your nick][register] -on Freenode, just because it's a good thing to do. - -You should also [subscribe][] to the Mercurial development [mailing list][]. -That's where you're going to be sending your work, so it's good to start -getting an idea of how people use the list as soon as possible. - -[irc]: irc://irc.freenode.net/#mercurial -[register]: http://freenode.net/faq.shtml#userregistration -[subscribe]: http://www.selenic.com/mailman/listinfo/mercurial-devel/ -[mailing list]: http://www.selenic.com/pipermail/mercurial-devel/ - -Grab the Source ---------------- - -Now you'll want to grab the Mercurial source code so you can start changing -it. The repository you almost certainly want to clone is the [crew -repository][]. - -Several people (listed [here][crew members]) have write access to this and -it's where most changes go before they're pulled into the main Mercurial -repositories. Your want your changes to apply cleanly to the tip of crew if -you want them to have the best chance of being accepted. - -Go ahead and clone it somewhere on your machine (replace `hg-crew` if you -prefer a different name for the folder): - - :::text - hg clone http://hg.intevation.org/mercurial/crew hg-crew - -It's going to take a little while. Mercurial might be fast but its repository -has over 8,600 changesets. While you're waiting you can move on to the next -step. - -[crew repository]: http://hg.intevation.org/mercurial/crew -[crew members]: http://www.selenic.com/mercurial/wiki/CrewRepository - -Set Up virtualenv ------------------ - -This might not seem like it's necessary, but trust me, it's a good thing to -do. It will make things easier down the road, I promise. - -No, *really*, take ten minutes and do this. You'll thank me. - -[virtualenv][] is a tool that lets you create separate Python development -environments that are isolated from each other. The packages and binaries for -one won't contaminate the others. It will make testing our work on Mercurial -painless. - -Open a new terminal window while the crew repository is busy cloning. Install -virtualenv in the usual Python fashion: - - :::text - sudo easy_install virtualenv - -Now install the [virtualenvwrapper][] tool: - - :::text - sudo easy_install virtualenvwrapper - -This is a nifty little shell script that makes working with virtual -environments much more pleasant. Before you can use it, you'll need to source -it into your `~/.bashrc` or `~/.bash_profile` file, as well as choose a place -to stick the environments themselves: - - :::bash - export WORKON_HOME=$HOME/lib/virtualenvs - source /(path to python)/bin/virtualenvwrapper_bashrc - -As you can see, I prefer to put the `virtualenvs` folder inside my `~/lib` -folder. You can change that to any place you like. - -You'll also need to replace the `(path to python)` with the actual path to -wherever Python is installed on your system. If you're not sure, here's a -simple, slow, and horribly inefficient but effective command to figure it out: - - :::text - find / -name 'virtualenvwrapper_bashrc' 2> /dev/null - -Once you've got the necessary lines in your `~/.bashrc` you can close that -terminal window. - -[virtualenv]: http://pypi.python.org/pypi/virtualenv -[virtualenvwrapper]: http://www.doughellmann.com/projects/virtualenvwrapper/ - -Set Up A Public Repository --------------------------- - -This isn't strictly required, but it's polite and only takes a minute or two. -If you've already got a method of sharing your repositories (using hgwebdir -with your website, for example) go ahead and use that. - -If not, head over to [BitBucket][], sign up for an account if you don't -already have one (it's free), and create a new public repository there. I've -named mine [hg-crew-sjl][] to be explicit about what it contains, but that's -not terribly important. - -Why bother doing this when you're just going to email your work anyway? Some -people (myself included) find it easier to pull changes from another -repository than to apply them from emails. It also gets your changes out into -a public place so other developers can view and modify *your* modifications. - -It's also nice to have an easily accessible clone of your personal repository -if you're not at your own machine and want to do some more work. - -[BitBucket]: http://bitbucket.org/ -[hg-crew-sjl]: http://bitbucket.org/sjl/hg-crew-sjl/ - -Set Up Your Local Clone ------------------------ - -By now the crew repository should be finished cloning to your local machine. -The first thing we want to do is set up the paths so we can push and pull -easily. Modify the `.hg/hgrc` file inside that repository (*not* your -`~/.hgrc` file!) to look like this: - - :::ini - [paths] - default = http://bitbucket.org/(username)/(repo name)/ - crew = http://hg.intevation.org/mercurial/crew - -Replace `(username)` with your BitBucket username and `(repo name)` with the -name of the BitBucket repository, obviously. - -Now you can run `hg push` to push any changes you commit locally to your -public BitBucket repository, and `hg pull crew` to grab new changes from the -crew repository. - -Build Mercurial ---------------- - -Make sure you're in your local crew repository, and run: - - :::text - make local - -This will build Mercurial from the source files. It should finish fairly -quickly. Once that's done, run the tests to make sure everything is working -correctly: - - :::text - make tests - -Those will take a long time to run. While you wait, move on to the next step. - -Create a Virtual Environment for Mercurial Development ------------------------------------------------------- - -While the test suite is running you're going to make a new virtual environment -to use for testing your changes to Mercurial. First you need to create it, so -open up a new terminal window and run: - - :::text - mkvirtualenv hg-dev - -You can name it something else if you don't like `hg-dev`. Notice how your -shell prompt now has `(hg-dev)` prepended to it? That's there to remind you -that you're working in that environment. - -Creating the environment automatically puts you into it for that shell -session. To get into it in the future you'll use: - - :::text - workon hg-dev - -Now you need to link the Mercurial libraries you're going to change to this -environment, so you can use it to test your code. Run the following three -commands: - - :::text - ln -s (full path to crew)/mercurial $WORKON_HOME/hg-dev/lib/python2.6/site-packages/ - ln -s (full path to crew)/hgext $WORKON_HOME/hg-dev/lib/python2.6/site-packages/ - ln -s (full path to crew)/hg $WORKON_HOME/hg-dev/bin/ - -Replace `(full path to crew)` with the *full* path to your local crew -repository, and `hg-dev` with the name of your virtual environment (if you -named it differently than I did). - -Why Bother With a Virtual Environment? --------------------------------------- - -It's probably the most work in this entire process, so why did I insist you -set it up? Watch this: - - :::console - sjl at ecgtheow in ~/src/hg-crew-sjl on default - $ hg --version - Mercurial Distributed SCM (version 1.2.1) - - Copyright (C) 2005-2009 Matt Mackall and others - This is free software; see the source for copying conditions. There is NO - warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - - sjl at ecgtheow in ~/src/hg-crew-sjl on default - $ workon hg-dev - - (hg-dev) - sjl at ecgtheow in ~/src/hg-crew-sjl on default - $ hg --version - Mercurial Distributed SCM (version 7956a823daa3) - - Copyright (C) 2005-2009 Matt Mackall and others - This is free software; see the source for copying conditions. There is NO - warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - - (hg-dev) - sjl at ecgtheow in ~/src/hg-crew-sjl on default - $ - -Look at the version numbers. Notice how they changed? - -**You can now use** `workon hg-dev` **to make the** `hg` **command run -Mercurial with your changes!** - -This makes it simple to test your work as you go. I keep two terminal windows -open while working on the Mercurial source: - -* One is in my local crew repository to diff, commit, etc with the standard - version of Mercurial. -* The other is in a test repository. I've run `workon hg-dev` to switch to my - modified version so I can try out my changes. - -Get To Work! ------------- - -That's it for the initial setup. Now you can start adding all the wonderful -features you have in mind. A simple workflow might go something like this: - -1. Open two terminal windows and use `workon hg-dev` in the second one to - switch to the virtual environment. -2. Make some changes to the Mercurial codebase. -3. Test your changes in the second terminal window. -4. Repeat 3 and 4, using the first terminal window to commit to the local - repository as you go. - -Once you're finished with your shiny new feature, how do you get it into the -main Mercurial code so we can all benefit from it? - -Modify the Test Suite ---------------------- - -Before you go off and submit a patch, you need to make sure it works with the -current tests. You can run the tests by going to the repository and using: - - :::text - cd tests - python run-tests.py - -It's going to take a while, just like before. If you changed something that -modifies the output of a command it will probably break some of the existing -tests. That's alright, but you'll need to fix them to reflect your changes. -The simplest way to do this is to use the `--interactive` flag: - - :::text - python run-tests.py --interactive - -Using `--interactive` will prompt you each time a test fails and ask if you -want to accept the new output as the "correct" output. This makes it simple to -update the existing tests to take your changes into account. - -Obviously you should **only accept these updates if they make sense**. Pay -attention to the output and make sure you didn't accidentally break something! - -Add to the Test Suite ----------------------- - -If you've implemented an entirely new feature (as opposed to just modifying an -existing one) you should add some tests of your own. This will make sure no -one else will inadvertently break whatever you've added. - -There's a nice [guide][test-guide] to this on the Mercurial site. Follow it to -add some tests for your changes. - -From what I've heard it's preferable to append to an existing set of tests (if -it makes sense) instead of adding a brand new test file -- the tests run -pretty slowly and adding new files make them even slower. - -[test-guide]: http://www.selenic.com/mercurial/wiki/WritingTests#Writing_a_shell_script_test - -Email Your Patches to the Mailing List --------------------------------------- - -You've finally got your code working and updated the tests. It's time to share -your changes with the rest of the Mercurial developers. - -If you want your patches to get accepted, you should take the time to read the -[guide to contributing changes][] and the [guide to making successful -patches][] on the Mercurial site. They're short but contain most of what you -need to know. - -[guide to contributing changes]: http://www.selenic.com/mercurial/wiki/ContributingChanges -[guide to making successful patches]: http://www.selenic.com/mercurial/wiki/SuccessfulPatch - -I prefer to use the [patchbomb][] extension to email the patches because it -takes care of the formatting for me. To enable and configure it you'll need to -make some additions to your `~/.hgrc` file: - - :::ini - [extensions] - hgext.patchbomb = - - [email] - method = smtp from = Your Name - - [smtp] - host = smtp.youremailhost.com username = yourusername tls = True - -You'll need to tweak those settings with your own email address, mail server, -and so on. Once that's done, you can have patchbomb package up your changes -and email them out: - - :::text - hg email --rev (your first revision):(your last revision) - -The command will guide you through filling out some more information, and then -send out the patches. `hg help email` has a lot more information on how to use -it. - -I usually email them to myself first just in case I mess up a revision number -and email six thousand changesets. Flooding my own inbox is much better than -flooding the list's. - -[patchbomb]: http://www.selenic.com/mercurial/wiki/PatchbombExtension - -That's It, Now Go Contribute! ------------------------------ - -That's about the extent of my knowledge on contributing to Mercurial. As I -said before, I've only done one patch so far, but I wanted to write this while -everything about setting up the environment was fresh in my mind. - -If any of the other Mercurial developers would like to chime in and suggest -corrections or additions -- or if anyone has any questions -- I'll be happy to -update the post. I hope it's helpful! - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2009/06/how-to-contribute-to-mercurial.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2009/06/how-to-contribute-to-mercurial.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,404 @@ ++++ +title = "How to Contribute to Mercurial" +snip = "Ten minutes of setup will make it easier." +date = 2009-06-01T20:09:44Z +draft = false + ++++ + +After my last post on [What I Hate About Mercurial][hate], seydar commented +that I should contribute a patch to add untrack/forget functionality. I +decided to take the advice, stop being lazy and write one. + +It's a very, very simple change and I'm not sure if it will even make it into +the core repository, but even so it gave me a chance to figure out how to set +up a decent development environment for hacking on Mercurial. I figured that +other people might find that information useful so I decided to write this. + +[hate]: http://stevelosh.com/blog/entry/2009/5/29/what-i-hate-about-mercurial/ + +{{% toc %}} + +About This Guide +---------------- + +This is meant to be a guide to get yourself set up to work on Mercurial's code +base and contribute your work back to the community. It's *not* a guide for +actually working on the code. + +I'm going to go ahead and assume a few (small) things: + +1. You're running OS X or some form of Linux. If you're using Windows, I'm + sorry, but I just don't use it enough to really help. +2. You've got [Python][] installed. +3. You've got a copy of Mercurial installed in some way (MacPorts, apt-get, + source, etc). +4. You're familiar with (and have clients for) IRC and email. + +If those things are true, let's get started. The entire process should take +between 15 and 45 minutes, depending on how comfortable you are with bash, +Python, and Mercurial itself. + +[Python]: {{links.python}} + +Join the Community +------------------ + +The first step to contributing is to join the community. Things will probably +go more smoothly if other developers have seen your name before and those +other developers are a great resource if you get stuck. + +Join the [#mercurial channel on Freenode][irc]. A lot of other Mercurial +developers hang out in there and it's a good place to go if you want some +really quick feedback. Take a few minutes and [register your nick][register] +on Freenode, just because it's a good thing to do. + +You should also [subscribe][] to the Mercurial development [mailing list][]. +That's where you're going to be sending your work, so it's good to start +getting an idea of how people use the list as soon as possible. + +[irc]: irc://irc.freenode.net/#mercurial +[register]: http://freenode.net/faq.shtml#userregistration +[subscribe]: http://www.selenic.com/mailman/listinfo/mercurial-devel/ +[mailing list]: http://www.selenic.com/pipermail/mercurial-devel/ + +Grab the Source +--------------- + +Now you'll want to grab the Mercurial source code so you can start changing +it. The repository you almost certainly want to clone is the [crew +repository][]. + +Several people (listed [here][crew members]) have write access to this and +it's where most changes go before they're pulled into the main Mercurial +repositories. Your want your changes to apply cleanly to the tip of crew if +you want them to have the best chance of being accepted. + +Go ahead and clone it somewhere on your machine (replace `hg-crew` if you +prefer a different name for the folder): + +```text +hg clone http://hg.intevation.org/mercurial/crew hg-crew +``` + +It's going to take a little while. Mercurial might be fast but its repository +has over 8,600 changesets. While you're waiting you can move on to the next +step. + +[crew repository]: http://hg.intevation.org/mercurial/crew +[crew members]: http://www.selenic.com/mercurial/wiki/CrewRepository + +Set Up virtualenv +----------------- + +This might not seem like it's necessary, but trust me, it's a good thing to +do. It will make things easier down the road, I promise. + +No, *really*, take ten minutes and do this. You'll thank me. + +[virtualenv][] is a tool that lets you create separate Python development +environments that are isolated from each other. The packages and binaries for +one won't contaminate the others. It will make testing our work on Mercurial +painless. + +Open a new terminal window while the crew repository is busy cloning. Install +virtualenv in the usual Python fashion: + +```text +sudo easy_install virtualenv +``` + +Now install the [virtualenvwrapper][] tool: + +```text +sudo easy_install virtualenvwrapper +``` + +This is a nifty little shell script that makes working with virtual +environments much more pleasant. Before you can use it, you'll need to source +it into your `~/.bashrc` or `~/.bash_profile` file, as well as choose a place +to stick the environments themselves: + +```bash +export WORKON_HOME=$HOME/lib/virtualenvs +source /(path to python)/bin/virtualenvwrapper_bashrc +``` + +As you can see, I prefer to put the `virtualenvs` folder inside my `~/lib` +folder. You can change that to any place you like. + +You'll also need to replace the `(path to python)` with the actual path to +wherever Python is installed on your system. If you're not sure, here's a +simple, slow, and horribly inefficient but effective command to figure it out: + +```text +find / -name 'virtualenvwrapper_bashrc' 2> /dev/null +``` + +Once you've got the necessary lines in your `~/.bashrc` you can close that +terminal window. + +[virtualenv]: http://pypi.python.org/pypi/virtualenv +[virtualenvwrapper]: http://www.doughellmann.com/projects/virtualenvwrapper/ + +Set Up A Public Repository +-------------------------- + +This isn't strictly required, but it's polite and only takes a minute or two. +If you've already got a method of sharing your repositories (using hgwebdir +with your website, for example) go ahead and use that. + +If not, head over to [BitBucket][], sign up for an account if you don't +already have one (it's free), and create a new public repository there. I've +named mine [hg-crew-sjl][] to be explicit about what it contains, but that's +not terribly important. + +Why bother doing this when you're just going to email your work anyway? Some +people (myself included) find it easier to pull changes from another +repository than to apply them from emails. It also gets your changes out into +a public place so other developers can view and modify *your* modifications. + +It's also nice to have an easily accessible clone of your personal repository +if you're not at your own machine and want to do some more work. + +[BitBucket]: http://bitbucket.org/ +[hg-crew-sjl]: http://bitbucket.org/sjl/hg-crew-sjl/ + +Set Up Your Local Clone +----------------------- + +By now the crew repository should be finished cloning to your local machine. +The first thing we want to do is set up the paths so we can push and pull +easily. Modify the `.hg/hgrc` file inside that repository (*not* your +`~/.hgrc` file!) to look like this: + +```ini +[paths] +default = http://bitbucket.org/(username)/(repo name)/ +crew = http://hg.intevation.org/mercurial/crew +``` + +Replace `(username)` with your BitBucket username and `(repo name)` with the +name of the BitBucket repository, obviously. + +Now you can run `hg push` to push any changes you commit locally to your +public BitBucket repository, and `hg pull crew` to grab new changes from the +crew repository. + +Build Mercurial +--------------- + +Make sure you're in your local crew repository, and run: + +```text +make local +``` + +This will build Mercurial from the source files. It should finish fairly +quickly. Once that's done, run the tests to make sure everything is working +correctly: + +```text +make tests +``` + +Those will take a long time to run. While you wait, move on to the next step. + +Create a Virtual Environment for Mercurial Development +------------------------------------------------------ + +While the test suite is running you're going to make a new virtual environment +to use for testing your changes to Mercurial. First you need to create it, so +open up a new terminal window and run: + +```text +mkvirtualenv hg-dev +``` + +You can name it something else if you don't like `hg-dev`. Notice how your +shell prompt now has `(hg-dev)` prepended to it? That's there to remind you +that you're working in that environment. + +Creating the environment automatically puts you into it for that shell +session. To get into it in the future you'll use: + +```text +workon hg-dev +``` + +Now you need to link the Mercurial libraries you're going to change to this +environment, so you can use it to test your code. Run the following three +commands: + +```text +ln -s (full path to crew)/mercurial $WORKON_HOME/hg-dev/lib/python2.6/site-packages/ +ln -s (full path to crew)/hgext $WORKON_HOME/hg-dev/lib/python2.6/site-packages/ +ln -s (full path to crew)/hg $WORKON_HOME/hg-dev/bin/ +``` + +Replace `(full path to crew)` with the *full* path to your local crew +repository, and `hg-dev` with the name of your virtual environment (if you +named it differently than I did). + +Why Bother With a Virtual Environment? +-------------------------------------- + +It's probably the most work in this entire process, so why did I insist you +set it up? Watch this: + +```console +sjl at ecgtheow in ~/src/hg-crew-sjl on default +$ hg --version +Mercurial Distributed SCM (version 1.2.1) + +Copyright (C) 2005-2009 Matt Mackall and others +This is free software; see the source for copying conditions. There is NO +warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + +sjl at ecgtheow in ~/src/hg-crew-sjl on default +$ workon hg-dev + +(hg-dev) +sjl at ecgtheow in ~/src/hg-crew-sjl on default +$ hg --version +Mercurial Distributed SCM (version 7956a823daa3) + +Copyright (C) 2005-2009 Matt Mackall and others +This is free software; see the source for copying conditions. There is NO +warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + +(hg-dev) +sjl at ecgtheow in ~/src/hg-crew-sjl on default +$ +``` + +Look at the version numbers. Notice how they changed? + +**You can now use** `workon hg-dev` **to make the** `hg` **command run +Mercurial with your changes!** + +This makes it simple to test your work as you go. I keep two terminal windows +open while working on the Mercurial source: + +* One is in my local crew repository to diff, commit, etc with the standard + version of Mercurial. +* The other is in a test repository. I've run `workon hg-dev` to switch to my + modified version so I can try out my changes. + +Get To Work! +------------ + +That's it for the initial setup. Now you can start adding all the wonderful +features you have in mind. A simple workflow might go something like this: + +1. Open two terminal windows and use `workon hg-dev` in the second one to + switch to the virtual environment. +2. Make some changes to the Mercurial codebase. +3. Test your changes in the second terminal window. +4. Repeat 3 and 4, using the first terminal window to commit to the local + repository as you go. + +Once you're finished with your shiny new feature, how do you get it into the +main Mercurial code so we can all benefit from it? + +Modify the Test Suite +--------------------- + +Before you go off and submit a patch, you need to make sure it works with the +current tests. You can run the tests by going to the repository and using: + +```text +cd tests +python run-tests.py +``` + +It's going to take a while, just like before. If you changed something that +modifies the output of a command it will probably break some of the existing +tests. That's alright, but you'll need to fix them to reflect your changes. +The simplest way to do this is to use the `--interactive` flag: + +```text +python run-tests.py --interactive +``` + +Using `--interactive` will prompt you each time a test fails and ask if you +want to accept the new output as the "correct" output. This makes it simple to +update the existing tests to take your changes into account. + +Obviously you should **only accept these updates if they make sense**. Pay +attention to the output and make sure you didn't accidentally break something! + +Add to the Test Suite +---------------------- + +If you've implemented an entirely new feature (as opposed to just modifying an +existing one) you should add some tests of your own. This will make sure no +one else will inadvertently break whatever you've added. + +There's a nice [guide][test-guide] to this on the Mercurial site. Follow it to +add some tests for your changes. + +From what I've heard it's preferable to append to an existing set of tests (if +it makes sense) instead of adding a brand new test file — the tests run +pretty slowly and adding new files make them even slower. + +[test-guide]: http://www.selenic.com/mercurial/wiki/WritingTests#Writing_a_shell_script_test + +Email Your Patches to the Mailing List +-------------------------------------- + +You've finally got your code working and updated the tests. It's time to share +your changes with the rest of the Mercurial developers. + +If you want your patches to get accepted, you should take the time to read the +[guide to contributing changes][] and the [guide to making successful +patches][] on the Mercurial site. They're short but contain most of what you +need to know. + +[guide to contributing changes]: http://www.selenic.com/mercurial/wiki/ContributingChanges +[guide to making successful patches]: http://www.selenic.com/mercurial/wiki/SuccessfulPatch + +I prefer to use the [patchbomb][] extension to email the patches because it +takes care of the formatting for me. To enable and configure it you'll need to +make some additions to your `~/.hgrc` file: + +```ini +[extensions] +hgext.patchbomb = + +[email] +method = smtp from = Your Name + +[smtp] +host = smtp.youremailhost.com username = yourusername tls = True +``` + +You'll need to tweak those settings with your own email address, mail server, +and so on. Once that's done, you can have patchbomb package up your changes +and email them out: + +```text +hg email --rev (your first revision):(your last revision) +``` + +The command will guide you through filling out some more information, and then +send out the patches. `hg help email` has a lot more information on how to use +it. + +I usually email them to myself first just in case I mess up a revision number +and email six thousand changesets. Flooding my own inbox is much better than +flooding the list's. + +[patchbomb]: http://www.selenic.com/mercurial/wiki/PatchbombExtension + +That's It, Now Go Contribute! +----------------------------- + +That's about the extent of my knowledge on contributing to Mercurial. As I +said before, I've only done one patch so far, but I wanted to write this while +everything about setting up the environment was fresh in my mind. + +If any of the other Mercurial developers would like to chime in and suggest +corrections or additions — or if anyone has any questions — I'll be happy to +update the post. I hope it's helpful! + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2009/08/a-guide-to-branching-in-mercurial.html --- a/content/blog/2009/08/a-guide-to-branching-in-mercurial.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,409 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "A Guide to Branching in Mercurial" - snip: "With illustrations and comparisons to git." - created: 2009-08-30 20:27:12 -%} - -{% block article_class %}with-diagrams{% endblock %} - -{% block article %} - -I've been hanging out in the [#mercurial][hg-irc] and [#bitbucket][bb-irc] -channels on freenode a lot lately, and I've noticed a topic that comes up a -lot is "how does [Mercurial][hg]'s branching differ from [git][]'s branching?" - -[hg]: {{links.mercurial}} -[git]: http://git-scm.com/ -[hg-irc]: irc://irc.freenode.net/#mercurial -[bb-irc]: irc://irc.freenode.net/#bitbucket -[freenode]: http://freenode.net/ - -A while ago [Nick Quaranto][nick] and I were talking about Mercurial and git's -branching models on Twitter and I wrote out a [quick longreply][lreply] about -the main differences. Since then I've pointed some git users toward that post -and they seemed to like it, so I figured I'd turn it into something a bit more -detailed. - -**Disclaimer:** this post is not intended to be a guide to the commands used -for working with Mercurial. It is only meant to be a guide to the *concepts* -behind the branching models. For more information on daily use and commands, -the [hg book][book] is a great resource (if you find it useful, please -[buy][book-buy] a paper copy to support Bryan and have a printed copy of the -[best editing fail of all time][editfail]). - -[nick]: http://litanyagainstfear.com/ -[lreply]: http://a.longreply.com/178502 -[book]: http://hgbook.red-bean.com/ -[book-buy]: http://search.barnesandnoble.com/Mercurial/Bryan-OSullivan/e/9780596800673/?itm=1 -[editfail]: http://twitpic.com/cwod4 - -[TOC] - -Prologue --------- - -Before I start explaining the different branching models, here's a simple -repository I'll use as an example: - -![Basic Repository Diagram](/media/images{{ parent_url }}/branch-base.png "Basic Repository") - -The repository is in the `~/src/test-project` folder. It has three changesets -in it: numbers 0, 1 and 2. - -**For git users:** each changeset in a Mercurial repository has a hash as an -identifier, just like with git. However, Mercurial also assigns numbers to -each changeset in a repository. The numbers are *only* for that local -repository -- two clones might have different numbers assigned to different -changesets depending on the order of pulls/pushes/etc. They're just there for -convenience while you're working with a repository. - -The default branch name in Mercurial is "default". Don't worry about what that -means for now, we'll get to it. I'm mentioning it because there's a little -`default` marker in the diagram. - -In all of these diagrams, a marker like that with a dashed border doesn't -actually exist as an object anywhere. Those are special names that you can use -to identify a changeset instead of the hash or number -- Mercurial will -calculate the revision on the fly. - -For now, ignore the `default` marker. I've colored it grey in each of the -diagrams where it doesn't matter. - -Branching with Clones ---------------------- - -The slowest, safest way to create a branch with Mercurial is to make a new -clone of the repository: - - :::console - $ cd ~/src - $ hg clone test-project test-project-feature-branch - -Now you've got two copies of the repository. You can commit separately in each -one and push/pull changesets between them as often as you like. Once you've -made some changes in each one, the result might look like this: - -![Clone Diagram](/media/images{{ parent_url }}/branch-clone.png "Branching with Clones") - -We've got two copies of the repository. Both contain the changesets that -existed at the time we branched/cloned. If we push from `test-project` into -`test-project-feature-branch` the "Fix a critical bug" changeset will be -pushed over. - -**For git users:** Remember how I mentioned that the changeset numbers are -local to a repository? We can see this clearly here -- there are two different -changesets with the number 3. The numbers are *only* used while working inside -a single repository. For pushing, pulling, or talking to other people you -should use the hashes. - -### Advantages - -Cloning is a very safe way of creating a branch. The two repositories are -completely isolated until you push or pull, so there's no danger of breaking -anything in one branch when you're working in another. - -Discarding a branch you don't want any more is *very* easy with cloned -branches. It's as simple as `rm -rf test-project-feature-branch`. There's no -need to mess around with editing repository history, you just delete the damn -thing. - -### Disadvantages - -Creating a branch by cloning locally is slower than the other methods, though -Mercurial will use hardlinks when cloning if your OS supports them (most do) -so it won't be *too* slow. - -However, the clone branching method can really slow things down when other -developers (not located nearby) want to work on the project. If you publish -two branches as separate repositories (such as `stable` and `version-2`), -contributors will have to clone down *both* repositories through the internet -if they want to work on both branches. That can take a lot of extra time, -depending on the repository size and bandwidth. - -It can become especially wasteful if, for example, there are 10,000 changesets -before the branch point and maybe 100 per branch after. Instead of pulling -down 10,200 changesets you need to pull down 20,200. If you want to work on -three different branches, you're pulling down 30,300 instead of 10,300. - -There is a way to avoid this large download cost, as pointed out by Guido -Ostkamp and timeless_mbp in [#mercurial][hg-irc]. The idea is that you clone -one branch down from the server, then pull *all* the branches into it, then -clone locally to split that repository back into branches. This avoids the -cost of cloning down the same changesets over and over. - -An example of this method with three branches would look something like this: - - :::console - $ hg clone http://server/project-main project - $ cd project - $ hg pull http://server/project-branch1 - $ hg pull http://server/project-branch2 - $ cd .. - $ hg clone project project-main --rev [head of mainline branch] - $ hg clone project project-branch1 --rev [head of branch1] - $ hg clone project project-branch2 --rev [head of branch2] - $ rm -rf project - $ cd project-main - $ [edit .hg/hgrc file to make the default path http://server/project-main] - $ cd ../project-branch1 - $ [edit .hg/hgrc to make the default path http://server/project-branch1] - $ cd ../project-branch2 - $ [edit .hg/hgrc to make the default path http://server/project-branch2] - -This example assumes you know the IDs of the branch heads off the top of your -head, which you probably don't. You'll have to look them up. - -It also assumes that there is only one new head per branch, when there might -be more. If `branch1` has two heads which are not in `mainline`, you would -need to look up the IDs of *both* and specify both in the clone command. - -Another annoyance shows up when you're working on a project that relies on -your code being at a specific file path. If you branch by cloning you'll need -to rename directories (or change the file path configuration) every time you -want to switch branches. This might not be a common situation (most build -tools don't care about the absolute path to the code) but it *does* appear now -and then. - -I personally don't like this method and don't use it. Others do though, so -it's good to understand it (Mercurial itself uses this model). - -### Comparison to git - -Git can use this method of branching too, although I don't see it very often. -Technically this is the exact same thing as creating a fork on [GitHub][], but -most people think of "fork" and "branch" as separate concepts. - -[GitHub]: http://github.com/ - -Branching with Bookmarks ------------------------- - -The next way to branch is to use a bookmark. For example: - - :::console - $ cd ~/src/test-project - $ hg bookmark main - $ hg bookmark feature - -[bookmark]: http://mercurial.selenic.com/wiki/BookmarksExtension - -Now you've got two bookmarks (essentially a tag) for your two branches at the -current changeset. - -To switch to one of these branches you can use `hg update feature` to update -to the tip changeset of that branch and mark yourself as working on that -branch. When you commit, it will move the bookmark to the newly created -changeset. - -**Note:** for more detailed information on actually using bookmarks day-to-day -please read the [bookmarks page][bookmark]. This guide is meant to show the -different branching models, and bookmarks have a few quirks that you should -know about if you're going to use them. - -Here's what the repository would look like with this method: - -![Bookmark Diagram](/media/images{{ parent_url }}/branch-bookmark.png "Branching with Bookmarks") - -The diagram of the changesets is pretty simple: the branch point was at -changeset 2 and each branch has one new changeset on it. - -Now let's look at the markers. The `default` marker is still there, and we're -still going to ignore it. - -There are two new labels in this diagram -- these represent the bookmarks. -Notice how their outlines are *not* dashed? This is because bookmarks are -actual objects stored on disk, not just convenient shortcuts that Mercurial -will let you use. - -When you use a bookmark name as a revision Mercurial will look up the revision -it points at and use that. - -### Advantages - -Bookmarks give you a quick, lightweight way to give meaningful labels to your -branches. - -You can delete them when you no longer need them. For example, if we finish -development of the new feature and merge the changes in the main branch, we -probably don't need to keep the `feature` bookmark around any more. - -### Disadvantages - -Being lightweight can also be a disadvantage. If you delete a bookmark, then -look at your code a year later and wonder what all those changesets that got -merged into main were for, the bookmark name is gone. This probably isn't a -big issue if you write good changeset summaries. - -Bookmarks are local. They do *not* get transferred during a push or pull! -There has been some whispering about adding this is Mercurial 1.4, but for now -if you want to give someone else your bookmarks you'll need to manually give -them the file the bookmarks are kept in. - -**UPDATE:** As of Mercurial 1.6 [bookmarks can be pushed and -pulled][remote-book] between repositories. - -[remote-book]: http://mercurial.selenic.com/wiki/BookmarksExtension#Working_With_Remote_Repositories - -### Comparison to git - -Branching with bookmarks is very close to the way git usually handles -branching. Mercurial bookmarks are like git refs: named pointers to changesets -that move on commit. - -The biggest difference is that git refs are transferred when pushing/pulling -and Mercurial bookmarks are not. - -Branching with Named Branches ------------------------------ - -The third way of branching is to use Mercurial's named branches. Some people -prefer this method (myself included) and many others don't. - -To create a new named branch: - - :::console - $ cd ~/src/test-project - $ hg branch feature - -When you commit the newly created changeset will be on the same branch as its -parent, unless you've used `hg branch` to mark it as being on a different one. - -Here's what a repository using named branches might look like: - -![Named Branch Diagram](/media/images{{ parent_url }}/branch-named.png "Branching with Named Branches") - -An important difference with this method is that the branch name is -permanently recorded as part of the changeset's metadata (as you can see in -changeset 4 in the diagram). - -**Note:** The default branch is called `default` and is not normally shown -unless you ask for verbose output. - -Now it's time to explain those magic dashed-border labels we've been ignoring. -Using a branch name to specify a revision is shorthand for "the tip changeset -of this named branch". In this example repository: - -* Running `hg update default` would update to changeset 3, which is the tip of - the `default` branch. -* Running `hg update feature` would update to changeset 4, which is the tip of - the `feature` branch. - -Neither of these labels actually exist as an object anywhere on disk (like a -bookmark would). When you use them Mercurial calculates the appropriate -revision on the fly. - -### Advantages - -The biggest advantage to using named branches is that every changeset on a -branch has the branch name as part of its metadata, which I find very helpful -(especially when using [graphlog][]). - -[graphlog]: http://mercurial.selenic.com/wiki/GraphlogExtension - -### Disadvantages - -Many people don't like cluttering up changeset metadata with branch names, -especially if they're small branches that are going to be merged pretty -quickly. - -In the past there was also the problem of not having a way to "close" a -branch, which means that over time the list of branches could get huge. This -was fixed in Mercurial 1.2 which introduced the `--close-branch` option for -`hg commit`. - -### Comparison to git - -As far as I know git has no equivalent to Mercurial's named branches. Branch -information is never stored as part of a git changeset's metadata. - -Branching Anonymously ---------------------- - -The last method of branching with Mercurial is the fastest and easiest: update -to any revision you want and commit. You don't have to think up a name for it -or do anything else -- just update and commit. - -When you update to a specific revision, Mercurial will mark the parent of the -working directory as that changeset. When you commit, the newly created -changeset's parent will be the parent of the working directory. - -The result of updating and committing without doing anything else would be: - -![Anonymous Diagram](/media/images{{ parent_url }}/branch-anon.png "Branching Anonymously") - -How do you switch back and forth between branches once you do this? Just use -`hg update --check REV` with the revision number (or hash) (you can shorten -`--check` to `-c`). - -**Note:** the `--check` option was added in Mercurial 1.3, but it was broken. -It's fixed in 1.3.1. If you're using something earlier than 1.3.1, you really -should update. - -Logging commands like `hg log` and `hg graphlog` will show you all the -changesets in the repository, so there's no danger of "losing" changesets. - -### Advantages - -This is the fastest, easiest way to branch. You don't have to think of a name -or close/delete anything when you're finished -- just update and commit. - -This method is *great* for quick-fix, two-or-three-changeset branches. - -### Disadvantages - -Using anonymous branching obviously means that there won't be any descriptive -name for a branch, so you'll need to write good commit messages if you want to -remember what a branch was for a couple of months later. - -Not having a single name to represent a branch means that you'll need to look -up the revision numbers or hashes with `hg log` or `hg graphlog` each time you -want to switch back and forth. If you're switching a lot this might be more -trouble than it's worth. - -### Comparison to git - -Git has no real way to handle this. Sure, it lets you update and commit, but -if you don't create a (named) ref to that new commit you're never going to -find it again once you switch to another one. Well, unless you feel like -grep'ing through a bunch of log output. - -Oh, and hopefully it doesn't get garbage collected. - -Sometimes you might not want to think up a name for a quick-fix branch. With -git you *have* to name it if you want to really do anything with it, with -Mercurial you don't. - -One More Difference Between Mercurial and git ---------------------------------------------- - -There's one more *big* difference between Mercurial's branching and git's -branching: - -**Mercurial will push/pull *all* branches by default, while git will push/pull -only the *current* branch.** - -This is important if you're a git user working with Mercurial. If you want to -push/pull only a single branch with Mercurial you can use the `--rev` option -(`-r` for short) and specify the tip revision of the branch: - - :::console - $ hg push --rev branchname - $ hg push --rev bookmarkname - $ hg push --rev 4 - -If you specify a revision, Mercurial will push that changeset and any -ancestors of it that the target doesn't already have. - -This doesn't apply when you use the "Branching with Clones" method because the -branches are separate repositories. - -Conclusion ----------- - -I hope this guide is helpful. If you see anything I've missed, especially on -the git side of things (I don't use git any more than I have to) or have any -questions please let me know! - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2009/08/a-guide-to-branching-in-mercurial.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2009/08/a-guide-to-branching-in-mercurial.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,409 @@ ++++ +title = "A Guide to Branching in Mercurial" +snip = "With illustrations and comparisons to git." +date = 2009-08-30T20:27:12Z +draft = false + ++++ + +I've been hanging out in the [#mercurial][hg-irc] and [#bitbucket][bb-irc] +channels on freenode a lot lately, and I've noticed a topic that comes up a +lot is "how does [Mercurial][hg]'s branching differ from [git][]'s branching?" + +[hg]: {{links.mercurial}} +[git]: http://git-scm.com/ +[hg-irc]: irc://irc.freenode.net/#mercurial +[bb-irc]: irc://irc.freenode.net/#bitbucket +[freenode]: http://freenode.net/ + +A while ago [Nick Quaranto][nick] and I were talking about Mercurial and git's +branching models on Twitter and I wrote out a [quick longreply][lreply] about +the main differences. Since then I've pointed some git users toward that post +and they seemed to like it, so I figured I'd turn it into something a bit more +detailed. + +**Disclaimer:** this post is not intended to be a guide to the commands used +for working with Mercurial. It is only meant to be a guide to the *concepts* +behind the branching models. For more information on daily use and commands, +the [hg book][book] is a great resource (if you find it useful, please +[buy][book-buy] a paper copy to support Bryan and have a printed copy of the +[best editing fail of all time][editfail]). + +[nick]: http://litanyagainstfear.com/ +[lreply]: http://a.longreply.com/178502 +[book]: http://hgbook.red-bean.com/ +[book-buy]: http://search.barnesandnoble.com/Mercurial/Bryan-OSullivan/e/9780596800673/?itm=1 +[editfail]: http://twitpic.com/cwod4 + +{{% toc %}} + +Prologue +-------- + +Before I start explaining the different branching models, here's a simple +repository I'll use as an example: + +Basic Repository + +The repository is in the `~/src/test-project` folder. It has three changesets +in it: numbers 0, 1 and 2. + +**For git users:** each changeset in a Mercurial repository has a hash as an +identifier, just like with git. However, Mercurial also assigns numbers to +each changeset in a repository. The numbers are *only* for that local +repository — two clones might have different numbers assigned to different +changesets depending on the order of pulls/pushes/etc. They're just there for +convenience while you're working with a repository. + +The default branch name in Mercurial is "default". Don't worry about what that +means for now, we'll get to it. I'm mentioning it because there's a little +`default` marker in the diagram. + +In all of these diagrams, a marker like that with a dashed border doesn't +actually exist as an object anywhere. Those are special names that you can use +to identify a changeset instead of the hash or number — Mercurial will +calculate the revision on the fly. + +For now, ignore the `default` marker. I've colored it grey in each of the +diagrams where it doesn't matter. + +Branching with Clones +--------------------- + +The slowest, safest way to create a branch with Mercurial is to make a new +clone of the repository: + +```console +$ cd ~/src +$ hg clone test-project test-project-feature-branch +``` + +Now you've got two copies of the repository. You can commit separately in each +one and push/pull changesets between them as often as you like. Once you've +made some changes in each one, the result might look like this: + +Branching with Clones + +We've got two copies of the repository. Both contain the changesets that +existed at the time we branched/cloned. If we push from `test-project` into +`test-project-feature-branch` the "Fix a critical bug" changeset will be +pushed over. + +**For git users:** Remember how I mentioned that the changeset numbers are +local to a repository? We can see this clearly here — there are two different +changesets with the number 3. The numbers are *only* used while working inside +a single repository. For pushing, pulling, or talking to other people you +should use the hashes. + +### Advantages + +Cloning is a very safe way of creating a branch. The two repositories are +completely isolated until you push or pull, so there's no danger of breaking +anything in one branch when you're working in another. + +Discarding a branch you don't want any more is *very* easy with cloned +branches. It's as simple as `rm -rf test-project-feature-branch`. There's no +need to mess around with editing repository history, you just delete the damn +thing. + +### Disadvantages + +Creating a branch by cloning locally is slower than the other methods, though +Mercurial will use hardlinks when cloning if your OS supports them (most do) +so it won't be *too* slow. + +However, the clone branching method can really slow things down when other +developers (not located nearby) want to work on the project. If you publish +two branches as separate repositories (such as `stable` and `version-2`), +contributors will have to clone down *both* repositories through the internet +if they want to work on both branches. That can take a lot of extra time, +depending on the repository size and bandwidth. + +It can become especially wasteful if, for example, there are 10,000 changesets +before the branch point and maybe 100 per branch after. Instead of pulling +down 10,200 changesets you need to pull down 20,200. If you want to work on +three different branches, you're pulling down 30,300 instead of 10,300. + +There is a way to avoid this large download cost, as pointed out by Guido +Ostkamp and timeless_mbp in [#mercurial][hg-irc]. The idea is that you clone +one branch down from the server, then pull *all* the branches into it, then +clone locally to split that repository back into branches. This avoids the +cost of cloning down the same changesets over and over. + +An example of this method with three branches would look something like this: + +```console +$ hg clone http://server/project-main project +$ cd project +$ hg pull http://server/project-branch1 +$ hg pull http://server/project-branch2 +$ cd .. +$ hg clone project project-main --rev [head of mainline branch] +$ hg clone project project-branch1 --rev [head of branch1] +$ hg clone project project-branch2 --rev [head of branch2] +$ rm -rf project +$ cd project-main +$ [edit .hg/hgrc file to make the default path http://server/project-main] +$ cd ../project-branch1 +$ [edit .hg/hgrc to make the default path http://server/project-branch1] +$ cd ../project-branch2 +$ [edit .hg/hgrc to make the default path http://server/project-branch2] +``` + +This example assumes you know the IDs of the branch heads off the top of your +head, which you probably don't. You'll have to look them up. + +It also assumes that there is only one new head per branch, when there might +be more. If `branch1` has two heads which are not in `mainline`, you would +need to look up the IDs of *both* and specify both in the clone command. + +Another annoyance shows up when you're working on a project that relies on +your code being at a specific file path. If you branch by cloning you'll need +to rename directories (or change the file path configuration) every time you +want to switch branches. This might not be a common situation (most build +tools don't care about the absolute path to the code) but it *does* appear now +and then. + +I personally don't like this method and don't use it. Others do though, so +it's good to understand it (Mercurial itself uses this model). + +### Comparison to git + +Git can use this method of branching too, although I don't see it very often. +Technically this is the exact same thing as creating a fork on [GitHub][], but +most people think of "fork" and "branch" as separate concepts. + +[GitHub]: http://github.com/ + +Branching with Bookmarks +------------------------ + +The next way to branch is to use a bookmark. For example: + +```console +$ cd ~/src/test-project +$ hg bookmark main +$ hg bookmark feature +``` + +[bookmark]: http://mercurial.selenic.com/wiki/BookmarksExtension + +Now you've got two bookmarks (essentially a tag) for your two branches at the +current changeset. + +To switch to one of these branches you can use `hg update feature` to update +to the tip changeset of that branch and mark yourself as working on that +branch. When you commit, it will move the bookmark to the newly created +changeset. + +**Note:** for more detailed information on actually using bookmarks day-to-day +please read the [bookmarks page][bookmark]. This guide is meant to show the +different branching models, and bookmarks have a few quirks that you should +know about if you're going to use them. + +Here's what the repository would look like with this method: + +Branching with Bookmarks + +The diagram of the changesets is pretty simple: the branch point was at +changeset 2 and each branch has one new changeset on it. + +Now let's look at the markers. The `default` marker is still there, and we're +still going to ignore it. + +There are two new labels in this diagram — these represent the bookmarks. +Notice how their outlines are *not* dashed? This is because bookmarks are +actual objects stored on disk, not just convenient shortcuts that Mercurial +will let you use. + +When you use a bookmark name as a revision Mercurial will look up the revision +it points at and use that. + +### Advantages + +Bookmarks give you a quick, lightweight way to give meaningful labels to your +branches. + +You can delete them when you no longer need them. For example, if we finish +development of the new feature and merge the changes in the main branch, we +probably don't need to keep the `feature` bookmark around any more. + +### Disadvantages + +Being lightweight can also be a disadvantage. If you delete a bookmark, then +look at your code a year later and wonder what all those changesets that got +merged into main were for, the bookmark name is gone. This probably isn't a +big issue if you write good changeset summaries. + +Bookmarks are local. They do *not* get transferred during a push or pull! +There has been some whispering about adding this is Mercurial 1.4, but for now +if you want to give someone else your bookmarks you'll need to manually give +them the file the bookmarks are kept in. + +**UPDATE:** As of Mercurial 1.6 [bookmarks can be pushed and +pulled][remote-book] between repositories. + +[remote-book]: http://mercurial.selenic.com/wiki/BookmarksExtension#Working_With_Remote_Repositories + +### Comparison to git + +Branching with bookmarks is very close to the way git usually handles +branching. Mercurial bookmarks are like git refs: named pointers to changesets +that move on commit. + +The biggest difference is that git refs are transferred when pushing/pulling +and Mercurial bookmarks are not. + +Branching with Named Branches +----------------------------- + +The third way of branching is to use Mercurial's named branches. Some people +prefer this method (myself included) and many others don't. + +To create a new named branch: + +```console +$ cd ~/src/test-project +$ hg branch feature +``` + +When you commit the newly created changeset will be on the same branch as its +parent, unless you've used `hg branch` to mark it as being on a different one. + +Here's what a repository using named branches might look like: + +Branching with Named Branches + +An important difference with this method is that the branch name is +permanently recorded as part of the changeset's metadata (as you can see in +changeset 4 in the diagram). + +**Note:** The default branch is called `default` and is not normally shown +unless you ask for verbose output. + +Now it's time to explain those magic dashed-border labels we've been ignoring. +Using a branch name to specify a revision is shorthand for "the tip changeset +of this named branch". In this example repository: + +* Running `hg update default` would update to changeset 3, which is the tip of + the `default` branch. +* Running `hg update feature` would update to changeset 4, which is the tip of + the `feature` branch. + +Neither of these labels actually exist as an object anywhere on disk (like a +bookmark would). When you use them Mercurial calculates the appropriate +revision on the fly. + +### Advantages + +The biggest advantage to using named branches is that every changeset on a +branch has the branch name as part of its metadata, which I find very helpful +(especially when using [graphlog][]). + +[graphlog]: http://mercurial.selenic.com/wiki/GraphlogExtension + +### Disadvantages + +Many people don't like cluttering up changeset metadata with branch names, +especially if they're small branches that are going to be merged pretty +quickly. + +In the past there was also the problem of not having a way to "close" a +branch, which means that over time the list of branches could get huge. This +was fixed in Mercurial 1.2 which introduced the `--close-branch` option for +`hg commit`. + +### Comparison to git + +As far as I know git has no equivalent to Mercurial's named branches. Branch +information is never stored as part of a git changeset's metadata. + +Branching Anonymously +--------------------- + +The last method of branching with Mercurial is the fastest and easiest: update +to any revision you want and commit. You don't have to think up a name for it +or do anything else — just update and commit. + +When you update to a specific revision, Mercurial will mark the parent of the +working directory as that changeset. When you commit, the newly created +changeset's parent will be the parent of the working directory. + +The result of updating and committing without doing anything else would be: + +Branching Anonymously + +How do you switch back and forth between branches once you do this? Just use +`hg update --check REV` with the revision number (or hash) (you can shorten +`--check` to `-c`). + +**Note:** the `--check` option was added in Mercurial 1.3, but it was broken. +It's fixed in 1.3.1. If you're using something earlier than 1.3.1, you really +should update. + +Logging commands like `hg log` and `hg graphlog` will show you all the +changesets in the repository, so there's no danger of "losing" changesets. + +### Advantages + +This is the fastest, easiest way to branch. You don't have to think of a name +or close/delete anything when you're finished — just update and commit. + +This method is *great* for quick-fix, two-or-three-changeset branches. + +### Disadvantages + +Using anonymous branching obviously means that there won't be any descriptive +name for a branch, so you'll need to write good commit messages if you want to +remember what a branch was for a couple of months later. + +Not having a single name to represent a branch means that you'll need to look +up the revision numbers or hashes with `hg log` or `hg graphlog` each time you +want to switch back and forth. If you're switching a lot this might be more +trouble than it's worth. + +### Comparison to git + +Git has no real way to handle this. Sure, it lets you update and commit, but +if you don't create a (named) ref to that new commit you're never going to +find it again once you switch to another one. Well, unless you feel like +grep'ing through a bunch of log output. + +Oh, and hopefully it doesn't get garbage collected. + +Sometimes you might not want to think up a name for a quick-fix branch. With +git you *have* to name it if you want to really do anything with it, with +Mercurial you don't. + +One More Difference Between Mercurial and git +--------------------------------------------- + +There's one more *big* difference between Mercurial's branching and git's +branching: + +**Mercurial will push/pull *all* branches by default, while git will push/pull +only the *current* branch.** + +This is important if you're a git user working with Mercurial. If you want to +push/pull only a single branch with Mercurial you can use the `--rev` option +(`-r` for short) and specify the tip revision of the branch: + +```console +$ hg push --rev branchname +$ hg push --rev bookmarkname +$ hg push --rev 4 +``` + +If you specify a revision, Mercurial will push that changeset and any +ancestors of it that the target doesn't already have. + +This doesn't apply when you use the "Branching with Clones" method because the +branches are separate repositories. + +Conclusion +---------- + +I hope this guide is helpful. If you see anything I've missed, especially on +the git side of things (I don't use git any more than I have to) or have any +questions please let me know! + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2009/11/my-sitesprint-project-lindyhub.html --- a/content/blog/2009/11/my-sitesprint-project-lindyhub.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,246 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "My SiteSprint Project: LindyHub" - snip: "I want to make something awesome for dancers." - created: 2009-11-16 19:15:07 -%} - -{% block article %} - -If you're a web designer and/or developer, you might have heard of -[SiteSprint][]. From the SiteSprint page: - ->What is the Site Sprint? ->A challenge to do something about our personal websites. If you were looking ->for a reason to rework an existing site or launch a new one, here it is. - -The event has a couple of rules: - ->The Rules ->1. Launch by 12/15 ->2. Document your process ->3. On launch, share what you did and how you did it - -I've decided to join in and finish up a pet project of mine that I've been -working on (very sporadically) for the past six months or so: [LindyHub][] - -[SiteSprint]: http://sitesprint.info/ -[LindyHub]: http://lindyhub.com/ - -[TOC] - -Preface: About Swing Dancing ----------------------------- - -If you're a dancer, go ahead and skip to the next section. This is for the -programmers and designers that might not have heard of Lindy Hop. - -[Lindy Hop][] is a form of swing dancing. I've been doing it for six years and -love it. - -Part of the Lindy Hop culture consists of "exchanges" and "workshop weekends". -The general idea is that a group of dancers will host one of these events on a -weekend. They'll hire DJs or bands, and teachers if they're going to have -classes, and plan out a whole weekend of awesomeness. - -Dancers from far and wide will travel to these events to meet and dance with -new people (and old friends). - -If you want more background on dancing find me on [Twitter][twsl] and I'll -gladly explain more, but this in-a-nutshell explanation should be enough for -you to understand the rest of this entry. - -[Lindy Hop]: http://en.wikipedia.org/wiki/Lindy_hop -[twsl]: {{links.twsl}} - -The Problem ------------ - -LindyHub is a project I've been working on for about six months off and on in -my spare time and it was rolling around in my head before that. - -Over the last few years or so I've been getting back into web development and -design. At the beginning of this year I started to notice a trend: most -websites for Lindy Hop and blues events are very basic and unpolished. Some of -the things I've noticed about almost every event site: - -* They have a form for you to register for the event, but the form almost - never has any nice inline validation of your input. -* Every site's form is different -- you have to read every single one - carefully to see exactly what they want. -* You end up entering the same information every single time for every event: - name, phone number, email, etc. -* Housing is a toss up. Sometimes they have a place for it in the registration - form, other times they just say "Email aaa@bbb.com if you need housing." - That's a lot of emails for the housing coordinator to sort through and quite - often it can become a mess. -* Some of them list addresses for each venue, which is great for those of us - that have GPSes. Sometimes they just give driving directions to and from - each one, which is terrible if you just want to type each one into your GPS - or print off directions from Google Maps before you go. -* Only one event that I've gone to ([Blues Muse][] in Philly) has had a "Who's - Coming" list so you could see who else was going to the event (you could - choose whether or not to let them show your name when you registered). - Sometimes you can figure this out from Facebook event pages, but that's - pretty unreliable. - -There are other things, but those are the big ones. - -Now, I completely understand why people don't add those features to their -event sites. It doesn't make sense to spend 10 hours adding cool features that -are nice to have but not really necessary if your website is only going to be -used for a single event. Those 10 hours could be spent promoting your event -and getting more people to come which is better for everyone. - -Around the time I was noticing these things I was also starting to get back -into contributing to open source projects. Two of the big project hosting -sites nowadays are [BitBucket][] and [GitHub][]. I noticed how they both make -starting a new project almost trivially easy -- you sign up for an account and -within two minutes people can view and contribute back to your new project. -They've fostered a lot of collaboration in the open source community and it's -been a very good thing all around. - -The two ideas came together in my head pretty quickly. I started thinking -about designing a site that made setting up a dance event almost as easy as -forking a project on BitBucket. - -Sure, every organizer spending 10 hours to add cool features to their event -site doesn't really make sense. But what if one person spent a couple hundred -hours and built a site that let every organizer add those features in 10 -*minutes*? After 20 or 30 events it's saved those couple hundred hours and -just gets better from there. - -The more I thought about it the more it seemed like a good idea. I'd been -learning a lot about web development and about six months ago I decided I was -competent enough to handle something like this, and started working on the -LindyHub core. - -[Blues Muse]: http://www.lindyandblues.com/events/2009/bluesmuse/ -[BitBucket]: http://bitbucket.org/ -[GitHub]: http://github.com/ - -LindyHub's Current State ------------------------- - -I've been working on LindyHub off and on for the past six months or so, and it -has been coming along pretty well. I decided to use SiteSprint as motivation -to buckle down and complete it to a point where I can invite a few organizers -to give it a try in a "closed-beta" period. From there I'll work out any kinks -and open it up for public use. - -Here are some of the things the site can do *right now*: - -* Users can sign up and create profiles for themselves. "Forgot my password" - and such is supported. -* They can view (and search for) events that are happening in the future, and - comment on them. -* They can create a new event and add things to its schedule. -* The schedule is formatted nicely. If an event starts before 5 AM it's - grouped with the previous day's events because it's probably a late night - dance. -* Venues will add links to Google Maps if the organizers put in the address. -* Organizers can let LindyHub handle event registration for them. They specify - how they want to be paid: door, check, and/or PayPal. If it's PayPal, - LindyHub takes the email address that should receive the money and handles - creating the PayPal "Buy Now" button. -* Early registration is supported -- organizers can specify a cutoff date and - the early/late prices if they like. -* LindyHub doesn't *have* to be used for the registration. Organizers can opt - to handle registration themselves and just put the event on LindyHub to help - more people find it. -* Users can register for events quickly and easily. LindyHub will ask them how - they want to pay and tell them how to do it. -* When they register they can allow other people to see that they're going to - the event. If they do, they'll show up in the list on the event page. -* Organizers can see nicely formatted lists of registration/payment - information and housing requests. - -My Goals for the Sprint (and Beyond) ------------------------------------- - -Some of the things I'm planning on adding to the site in during the sprint -are: - -* Redesign the interface. I'm not a designer, but I can certainly do better - than the spartan design I've thrown together during development. -* Better, Javascript-based form fields. They'll validate the input before you - even click "Submit" (so you can correct mistakes immediately) and provide - nice date/time pickers for the date and time fields. I wanted to make sure - the site works 100% for people with JavaScript turned off before I started - relying on it to make things pretty. -* Mobile stylesheets for iPhone users. -* Tagging events with tags like "lindyhop, blues, workshop, exchange, - gordonwebster, etc." This will make it easier to find events you're - interested in. This will also be easy to add. -* "Friending" other users. This would let you see what your friends are going - to and would be awesome (for me, at least). -* Location-based features. LindyHub can already store your hometown and the - locations of events. It wouldn't be terribly difficult to parse this - information into a latitude and longitude. Once that's done it's simple to - calculate the distance to events and say "Show me all events within 300 - miles of my hometown" or "Show me everyone going to this event that lives - within 30 miles of me because we might want to carpool." -* Automatically email reminders to people a week before the event. The emails - could include links to Google Maps for the first (and other) venues to make - it take only one click to print directions. There are a lot of other things - that would be nice to include too -- I've been holding off on this because - it really requires the attention of a copywriter to get right. -* Provide an easy-to-use interface for organizers to assign people that have - requested housing to people that offer it. -* Send emails to guest/host groups automatically to make sure guests have - their host's contact information (and vice versa). -* Translations into different languages. This will be pretty simple if someone - wants to volunteer to translate (I only speak English and American Sign - Language myself). -* Suggesting events a user might like. This is something for the far future, - but it would give me a chance to play around in another favorite area of - mine: artificial intelligence. I have some ideas on how to implement it but - it's not critical so I'm not looking at it right now. - -There are a whole slew of other things I have tumbling about in my head, but -those should be enough to give you an idea of where the site can go. - -**Above all, my goal is to spend as much time as it takes to create a site -that will save organizers a few hours of their time and make it easier for -dancers to find events they want to go to.** - -I *don't* want to try to create a site that will accommodate *every* event. If -I make LindyHub flexible enough to handle Frankie 95 it will become so -complicated that no other events will want to use it. I want it to work for -the 85% or 90% of events that just need the most common features. Doing that -will make it far, far easier to use, which means more people will use it. - -For the Nerds: What I'm Using ------------------------------ - -LindyHub is built with [Django][] and [Python][]. - -It's hosted on [WebFaction][] and uses a [PostgreSQL][] database to store its -data. - -It uses [Aardvark Legs][] to keep me sane while writing the CSS, and will use -[jQuery][] for the inline form validation. - -I'm happy to go further in depth if you're curious -- find me on [Twitter][twsl]. - -[Django]: {{links.django}} -[Python]: {{links.python}} -[WebFaction]: {{links.webfaction}} -[PostgreSQL]: http://www.postgresql.org/ -[Aardvark Legs]: {{links.aardvarklegs}} -[jQuery]: http://jquery.com/ - -If You're Interested, Follow My Progress! ------------------------------------------ - -I'm going to be working on the site a lot over the next month. I might write -another blog entry or two, but I'll try to post updates to -[@lindyhub][twitter] on Twitter much more often. - -If there's anything specific you want to know find me on [Twitter][twsl] and -I'll be happy to answer! - -[twitter]: http://twitter.com/lindyhub/ - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2009/11/my-sitesprint-project-lindyhub.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2009/11/my-sitesprint-project-lindyhub.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,243 @@ ++++ +title = "My SiteSprint Project: LindyHub" +snip = "I want to make something awesome for dancers." +date = 2009-11-16T19:15:07Z +draft = false + ++++ + +If you're a web designer and/or developer, you might have heard of +[SiteSprint][]. From the SiteSprint page: + +>What is the Site Sprint? +>A challenge to do something about our personal websites. If you were looking +>for a reason to rework an existing site or launch a new one, here it is. + +The event has a couple of rules: + +>The Rules +>1. Launch by 12/15 +>2. Document your process +>3. On launch, share what you did and how you did it + +I've decided to join in and finish up a pet project of mine that I've been +working on (very sporadically) for the past six months or so: [LindyHub][] + +[SiteSprint]: http://sitesprint.info/ +[LindyHub]: http://lindyhub.com/ + +{{% toc %}} + +Preface: About Swing Dancing +---------------------------- + +If you're a dancer, go ahead and skip to the next section. This is for the +programmers and designers that might not have heard of Lindy Hop. + +[Lindy Hop][] is a form of swing dancing. I've been doing it for six years and +love it. + +Part of the Lindy Hop culture consists of "exchanges" and "workshop weekends". +The general idea is that a group of dancers will host one of these events on a +weekend. They'll hire DJs or bands, and teachers if they're going to have +classes, and plan out a whole weekend of awesomeness. + +Dancers from far and wide will travel to these events to meet and dance with +new people (and old friends). + +If you want more background on dancing find me on [Twitter][twsl] and I'll +gladly explain more, but this in-a-nutshell explanation should be enough for +you to understand the rest of this entry. + +[Lindy Hop]: http://en.wikipedia.org/wiki/Lindy_hop +[twsl]: {{links.twsl}} + +The Problem +----------- + +LindyHub is a project I've been working on for about six months off and on in +my spare time and it was rolling around in my head before that. + +Over the last few years or so I've been getting back into web development and +design. At the beginning of this year I started to notice a trend: most +websites for Lindy Hop and blues events are very basic and unpolished. Some of +the things I've noticed about almost every event site: + +* They have a form for you to register for the event, but the form almost + never has any nice inline validation of your input. +* Every site's form is different — you have to read every single one + carefully to see exactly what they want. +* You end up entering the same information every single time for every event: + name, phone number, email, etc. +* Housing is a toss up. Sometimes they have a place for it in the registration + form, other times they just say "Email aaa@bbb.com if you need housing." + That's a lot of emails for the housing coordinator to sort through and quite + often it can become a mess. +* Some of them list addresses for each venue, which is great for those of us + that have GPSes. Sometimes they just give driving directions to and from + each one, which is terrible if you just want to type each one into your GPS + or print off directions from Google Maps before you go. +* Only one event that I've gone to ([Blues Muse][] in Philly) has had a "Who's + Coming" list so you could see who else was going to the event (you could + choose whether or not to let them show your name when you registered). + Sometimes you can figure this out from Facebook event pages, but that's + pretty unreliable. + +There are other things, but those are the big ones. + +Now, I completely understand why people don't add those features to their +event sites. It doesn't make sense to spend 10 hours adding cool features that +are nice to have but not really necessary if your website is only going to be +used for a single event. Those 10 hours could be spent promoting your event +and getting more people to come which is better for everyone. + +Around the time I was noticing these things I was also starting to get back +into contributing to open source projects. Two of the big project hosting +sites nowadays are [BitBucket][] and [GitHub][]. I noticed how they both make +starting a new project almost trivially easy — you sign up for an account and +within two minutes people can view and contribute back to your new project. +They've fostered a lot of collaboration in the open source community and it's +been a very good thing all around. + +The two ideas came together in my head pretty quickly. I started thinking +about designing a site that made setting up a dance event almost as easy as +forking a project on BitBucket. + +Sure, every organizer spending 10 hours to add cool features to their event +site doesn't really make sense. But what if one person spent a couple hundred +hours and built a site that let every organizer add those features in 10 +*minutes*? After 20 or 30 events it's saved those couple hundred hours and +just gets better from there. + +The more I thought about it the more it seemed like a good idea. I'd been +learning a lot about web development and about six months ago I decided I was +competent enough to handle something like this, and started working on the +LindyHub core. + +[Blues Muse]: http://www.lindyandblues.com/events/2009/bluesmuse/ +[BitBucket]: http://bitbucket.org/ +[GitHub]: http://github.com/ + +LindyHub's Current State +------------------------ + +I've been working on LindyHub off and on for the past six months or so, and it +has been coming along pretty well. I decided to use SiteSprint as motivation +to buckle down and complete it to a point where I can invite a few organizers +to give it a try in a "closed-beta" period. From there I'll work out any kinks +and open it up for public use. + +Here are some of the things the site can do *right now*: + +* Users can sign up and create profiles for themselves. "Forgot my password" + and such is supported. +* They can view (and search for) events that are happening in the future, and + comment on them. +* They can create a new event and add things to its schedule. +* The schedule is formatted nicely. If an event starts before 5 AM it's + grouped with the previous day's events because it's probably a late night + dance. +* Venues will add links to Google Maps if the organizers put in the address. +* Organizers can let LindyHub handle event registration for them. They specify + how they want to be paid: door, check, and/or PayPal. If it's PayPal, + LindyHub takes the email address that should receive the money and handles + creating the PayPal "Buy Now" button. +* Early registration is supported — organizers can specify a cutoff date and + the early/late prices if they like. +* LindyHub doesn't *have* to be used for the registration. Organizers can opt + to handle registration themselves and just put the event on LindyHub to help + more people find it. +* Users can register for events quickly and easily. LindyHub will ask them how + they want to pay and tell them how to do it. +* When they register they can allow other people to see that they're going to + the event. If they do, they'll show up in the list on the event page. +* Organizers can see nicely formatted lists of registration/payment + information and housing requests. + +My Goals for the Sprint (and Beyond) +------------------------------------ + +Some of the things I'm planning on adding to the site in during the sprint +are: + +* Redesign the interface. I'm not a designer, but I can certainly do better + than the spartan design I've thrown together during development. +* Better, Javascript-based form fields. They'll validate the input before you + even click "Submit" (so you can correct mistakes immediately) and provide + nice date/time pickers for the date and time fields. I wanted to make sure + the site works 100% for people with JavaScript turned off before I started + relying on it to make things pretty. +* Mobile stylesheets for iPhone users. +* Tagging events with tags like "lindyhop, blues, workshop, exchange, + gordonwebster, etc." This will make it easier to find events you're + interested in. This will also be easy to add. +* "Friending" other users. This would let you see what your friends are going + to and would be awesome (for me, at least). +* Location-based features. LindyHub can already store your hometown and the + locations of events. It wouldn't be terribly difficult to parse this + information into a latitude and longitude. Once that's done it's simple to + calculate the distance to events and say "Show me all events within 300 + miles of my hometown" or "Show me everyone going to this event that lives + within 30 miles of me because we might want to carpool." +* Automatically email reminders to people a week before the event. The emails + could include links to Google Maps for the first (and other) venues to make + it take only one click to print directions. There are a lot of other things + that would be nice to include too — I've been holding off on this because + it really requires the attention of a copywriter to get right. +* Provide an easy-to-use interface for organizers to assign people that have + requested housing to people that offer it. +* Send emails to guest/host groups automatically to make sure guests have + their host's contact information (and vice versa). +* Translations into different languages. This will be pretty simple if someone + wants to volunteer to translate (I only speak English and American Sign + Language myself). +* Suggesting events a user might like. This is something for the far future, + but it would give me a chance to play around in another favorite area of + mine: artificial intelligence. I have some ideas on how to implement it but + it's not critical so I'm not looking at it right now. + +There are a whole slew of other things I have tumbling about in my head, but +those should be enough to give you an idea of where the site can go. + +**Above all, my goal is to spend as much time as it takes to create a site +that will save organizers a few hours of their time and make it easier for +dancers to find events they want to go to.** + +I *don't* want to try to create a site that will accommodate *every* event. If +I make LindyHub flexible enough to handle Frankie 95 it will become so +complicated that no other events will want to use it. I want it to work for +the 85% or 90% of events that just need the most common features. Doing that +will make it far, far easier to use, which means more people will use it. + +For the Nerds: What I'm Using +----------------------------- + +LindyHub is built with [Django][] and [Python][]. + +It's hosted on [WebFaction][] and uses a [PostgreSQL][] database to store its +data. + +It uses [Aardvark Legs][] to keep me sane while writing the CSS, and will use +[jQuery][] for the inline form validation. + +I'm happy to go further in depth if you're curious — find me on [Twitter][twsl]. + +[Django]: {{links.django}} +[Python]: {{links.python}} +[WebFaction]: {{links.webfaction}} +[PostgreSQL]: http://www.postgresql.org/ +[Aardvark Legs]: {{links.aardvarklegs}} +[jQuery]: http://jquery.com/ + +If You're Interested, Follow My Progress! +----------------------------------------- + +I'm going to be working on the site a lot over the next month. I might write +another blog entry or two, but I'll try to post updates to +[@lindyhub][twitter] on Twitter much more often. + +If there's anything specific you want to know find me on [Twitter][twsl] and +I'll be happy to answer! + +[twitter]: http://twitter.com/lindyhub/ + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2010/01/moving-from-django-to-hyde.html --- a/content/blog/2010/01/moving-from-django-to-hyde.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,461 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "Moving from Django to Hyde" - snip: "Another year, another rewrite." - created: 2010-01-15 20:14:00 -%} - -{% block article %} - -Almost exactly one year ago I posted a blog entry about how I [rewrote this -site][rewrite] using [Django][]. It's a new year, and once again I've -rewritten the site. - -If you've visited my site before you may have noticed some small style tweaks. -I've cleaned up a few things and I think the site looks better overall. The -biggest change, however, is that I completely rewrote the core -- this time -with [Hyde][]. - -Hyde is a "static site generator" written in [Python][], similar to -[Jekyll][], [Blatter][] and [Pilcrow][]. I chose it over the others because -it's written in Python (unlike Jekyll), more flexible than Blatter and more -mature than Pilcrow. - -Rewriting an existing site that gets a decent amount of visitors is different -than creating a new site from scratch, so I decided to write this entry to -describe some of the issues I ran into and how I tackled them. - -[Django]: {{links.django}} -[rewrite]: /blog/2009/01/site-redesign/ -[Hyde]: {{links.hyde}} -[Python]: {{links.python}} -[Jekyll]: http://jekyllrb.com/ -[Blatter]: {{links.blatter}} -[Pilcrow]: http://inky.github.com/pilcrow/ - -[TOC] - -Why Static? ------------ - -There are a couple of reasons I decided to switch to a static site. - -### Less Memory Needed - -I use [WebFaction][] for my web hosting, and with my current plan I have a -limit on how much memory I can use at any given time. Each Django site takes -up around 20mb of memory on the server. - -By switching to a static set of HTML pages for my site I save those 20mb so I -can use them for something else (like a staging site for another project). My -site doesn't particularly *need* all of the functionality Django can provide, -so I figured I'd switch to a static site and save the memory. - -[WebFaction]: {{links.webfaction}} - -### Static Pages are Faster - -My site doesn't get an enormous amount of traffic, so the Django instances -weren't really working very hard. Still, serving a static page through Apache -or nginx is always going to be faster than running it through Django. -Faster-loading pages is always a good thing, even if the speed increase isn't -very large. - -### Easier to Maintain - -This is the main reason I switched. Previously, to edit a blog entry I would -log in through Django's admin interface and edit the entry. With Hyde, I -simply edit a file (or create a new one) on my machine and then run a single -command to publish it. - -Another problem with the old site is that I needed to be connected to the -internet to get to the admin interface, so I couldn't update entries (or -publish new ones) offline. I usually have an internet connection, but -occasionally I don't. Now I can edit as much as I like on my own machine and -only need a connection to publish the finished product. - -### Easier to Backup - -One more advantage of Hyde is that the site structure *and* content are all -held in the same place. I keep everything in a [Mercurial][] repository, and -so every time I push that repository somewhere it creates a full backup of the -site's code *and* content. If WebFaction's server catches on fire I still have -everything on my local machine (and on BitBucket). - -I've toyed around with backing up Django's database tables when I had my old -site, but this new method is far less work. - -[Mercurial]: {{links.mercurial}} - -Getting Started ---------------- - -I wanted a fresh start when I was rewriting the site, so I went ahead and -created a brand new folder and Mercurial repository for it. - -I used `hyde --init` to lay out a skeleton in the new folder. Then I stripped -out a lot of the default items that get added with `--init` and created the -directory structure I wanted to use, with stubs for each of the main content -files I knew I'd need. - -Finally, I went ahead and filled some of the basic values in the `settings.py` -file. - -Layout and Styling ------------------- - -Once I had a skeleton in place I started working on layout of the site. - -### Cleanup - -The templates created by `hyde --init` are functional, but when you look at -the code they're a mess. The indentation is strange and inconsistent and -there's trailing whitespace all over the place. I like clean code so I sat -down and cleaned everything up before I started making any real changes. - -### Rewriting the Layout and Styles - -After I finished cleaning up the templates I duplicated the HTML structure and -styles of the old site from scratch. The old site had gone through a bunch of -iterations and I was a bit sloppy in my editing, so there was a lot of cruft -that had snuck in. I wanted a truly *fresh* start for the site, so I buckled -down and did it all again. - -### A New CSS Framework - -Previously I used the [Blueprint CSS framework][blueprint] to make laying out -the site easier. Blueprint is a great framework, but it's more powerful than I -need for a site as simple as this one. The site now uses [Aardvark Legs][aal], -which is a much simpler framework that simply sets up a great vertical rhythm -and leaves you free to lay out the horizontal structure yourself. - -### Pagination - -Before the rewrite the list of blog entries used to be paginated. Hyde -supports pagination but I decided against using it because I simply don't -write enough to make it necessary. All the blog entries are now listed on a -single page, which means that you can use Cmd+F to find an article if you're -looking for something specific. - -[blueprint]: http://www.blueprintcss.org/ -[aal]: {{links.aardvarklegs}} - -Using Fabric to Type Less -------------------------- - -I realized very quickly that typing `hyde -g -s . && hyde -w -s .` would get -old pretty quickly, so I installed [Fabric][] and wrote a fabfile. - -Fabric is a tool written in Python that lets you define tasks and execute them -by running `fab taskname`. Fabfiles are pure Python, so you can build larger -tasks out of smaller ones very easily and do just about anything you want. -It's similar to [ant][], but without the excessive over-engineering. - -You can view the [current fabfile][fabfile] on BitBucket. At the time of this -entry it looks like this: - - :::python - from fabric.api import * - import os - import fabric.contrib.project as project - - PROD = 'sjl.webfactional.com' - DEST_PATH = '/home/sjl/webapps/slc/' - ROOT_PATH = os.path.abspath(os.path.dirname(__file__)) - DEPLOY_PATH = os.path.join(ROOT_PATH, 'deploy') - - def clean(): - local('rm -rf ./deploy') - - def regen(): - clean() - local('hyde -g -s .') - - def serve(): - local('hyde -w -s .') - - def reserve(): - regen() - serve() - - def smush(): - local('smusher ./media/images') - - @hosts(PROD) - def publish(): - regen() - project.rsync_project( - remote_dir=DEST_PATH, - local_dir=DEPLOY_PATH.rstrip('/') + '/', - delete=True - ) - -The task I use most often while developing is `fab reserve`, which regenerates -the site and then starts serving it so I can view the result in a browser. - -I use `fab smush` whenever I add new images. This runs [smusher][] on all of -the images to optimize them without reducing quality. - -When I'm ready to publish changes to the live site I run `fab publish`, which -will regenerate my local version and copy it up to the WebFaction server. - -[Fabric]: {{links.fabric}} -[ant]: http://ant.apache.org/ -[fabfile]: http://bitbucket.org/sjl/stevelosh/src/tip/fabfile.py -[smusher]: http://github.com/grosser/smusher - -Moving the Content ------------------- - -The content of the old site (blog entries, projects, and static pages like the -[about page][]) was fairly easy to migrate over to the new one, because both -versions use [Markdown][] to format the text. - -First I created an empty file for each page with a filename that matched the -"slug" (last part of the URL) of the old page. Then I manually copied over the -title, creation time and content for every page. I could have written a script -to do this, but I don't have enough pages on the site to make it worth the -time. - -[about page]: /about/ -[Markdown]: {{links.markdown}} - -Converting the Comments ------------------------ - -At this point the new site was looking very much like the old one. The styles -were (roughly) matching and the blog posts, entries, and static pages were all -rendered nicely. - -The next big task was migrating all of the comments. Since the new site is -static it can't handle dynamic content like comments on its own, so I decided -to use [Disqus][]. I use Disqus for the comments on the [hg tip][] site and -it's very nice. For that site I used it from the beginning, but for this -rewrite I needed to somehow import all the old comments. - -To migrate the comments over I used [django-disqus][], but there was a small -snag that I needed to deal with first. - -When I first wrote the old site I was just getting back into Django after not -using it for a long time. I didn't know about Django's built-in comment -models, so I created my own. They weren't as good as Django's, but they did -the job and I didn't care enough to change them. - -This became a problem when it was time to import the old comments into Disqus. -django-disqus only supports importing comments from Django's built-in comment -models. To work around this I first had to convert the old comments into -Django's built-in ones. I wrote a [small, hacky Python -script][convert-comments] to do it: - - :::python - #!/usr/bin/env python - - from django.core.management import setup_environ - import settings - setup_environ(settings) - - from markdown import Markdown - from django.contrib.comments.models import Comment - from django.contrib.sites.models import Site - from stevelosh.blog.models import Comment as BlogComment - from stevelosh.projects.models import Comment as ProjectComment - - - mdown = Markdown() - - site = Site.objects.all()[0] - blog_comments = BlogComment.objects.filter(spam=False) - project_comments = ProjectComment.objects.filter(spam=False) - - for bc in blog_comments: - c = Comment() - c.content_object = bc.entry - c.user_name = bc.name - c.comment = mdown.convert(bc.body) - c.submit_date = bc.submitted - c.site = site - c.is_public = True - c.is_removed = False - c.save() - # print 'http://%s%s' % (site.domain, c.content_object.get_absolute_url()) - - for pc in project_comments: - c = Comment() - c.content_object = pc.project - c.user_name = pc.name - c.comment = mdown.convert(pc.body) - c.submit_date = pc.submitted - c.site = site - c.is_public = True - c.is_removed = False - c.save() - # print 'http://%s%s' % (site.domain, c.content_object.get_absolute_url()) - -Yes, it's ugly. No, I don't care. It was run once or twice locally and once on -the live server. It worked and I'll never need to run it again. - -Once I had the comments converted I could use django-disqus to migrate them. -The import went very smoothly -- I ran one command and after a couple of -minutes everything was in Disqus. Once that finished it was just a matter of -adding the Disqus JavaScript to one of the templates. - -[Disqus]: http://disqus.com/ -[hg tip]: {{links.hgtip}} -[django-disqus]: http://github.com/arthurk/django-disqus -[convert-comments]: http://bitbucket.org/sjl/stevelosh/src/da98105753a1/convert-comments.py - -Rewriting the Old URLs ----------------------- - -Since I was pretty much starting from scratch with this rewrite I decided to -clean up the URL structure of my blog. Previously a blog entry's URL looked -like this: - - :::text - http://stevelosh.com/blog/entry/2009/08/30/a-guide-to-branching-in-mercurial/ - -The `/entry/` part of the URL was useless -- it just took up space, so I got -rid of it. The year and month are useful to get an idea of how old an entry is -just by looking at the link, so I left them in. The day, however, probably -doesn't matter that much, so I took it out. - -The new URLs look like this: - - :::text - http://stevelosh.com/blog/2009/08/a-guide-to-branching-in-mercurial/ - -The problem with rewriting the URL structure is that there are already links -around the web pointing at my entries. I didn't want those old links to break, -so I crafted an [`.htaccess` file][htaccess] that would rewrite the old URLs -into the new ones: - - {% templatetag openblock %} if GENERATE_CLEAN_URLS {% templatetag closeblock %} - RewriteEngine on - RewriteBase {% templatetag openvariable %} node.site.settings.SITE_ROOT {% templatetag closevariable %} - - # Old URLs - RewriteRule ^blog/entry/(\d+)/(\d\d)/\d+/([^/]*)/?$ /blog/$1/$2/$3/ [R=301,L] - RewriteRule ^blog/entry/(\d+)/(\d)/\d+/([^/]*)/?$ /blog/$1/0$2/$3/ [R=301,L] - - {% templatetag openblock %} hyde_listing_page_rewrite_rules {% templatetag closeblock %} - - # listing pages whose names are the same as their enclosing folder's - RewriteCond %{REQUEST_FILENAME}/$1.html -f - RewriteRule ^([^/]*)/$ %{REQUEST_FILENAME}/$1.html - - # regular pages - RewriteCond %{REQUEST_FILENAME}.html -f - RewriteRule ^.*$ %{REQUEST_FILENAME}.html - - {% templatetag openblock %} endif {% templatetag closeblock %} - -Most of that file is the stock Hyde `.htaccess` file -- the two lines under `# -Old URLs` are the ones that I added. It took me a while to get it right -because I don't work with Apache's `mod_rewrite` very often, but it was worth -it to avoid breaking all of the old links. - -[htaccess]: http://bitbucket.org/sjl/stevelosh/src/tip/content/.htaccess - -Creating an RSS Feed --------------------- - -I had an RSS feed for the old site which Django made very easy to set up. I -definitely needed a feed for the new site, and fortunately Hyde provides a -simple sample template that can be used to make an ATOM feed. - -I cleaned up the whitespace and formatting of that template a bit, adjusted a -few variables for my needs, put it on [FeedBurner][] and everything was ready. - -[FeedBurner]: http://www.feedburner.com/ - -Merging the Repositories ------------------------- - -The last step before I was finished was to merge the old and new repositories -together so I wouldn't lose any of the history. It's probably not *too* -important to keep the old site's history around, but I put a lot of work into -it over the past year and it has some sentimental value. - -Fortunately it's very easy to [combine Mercurial repositories][combinerepos]. -I just pulled the old repository into the new one, merged the old head into -the new one while discarding all the changes, and pushed it up to BitBucket. - -[combinerepos]: http://hgtip.com/tips/advanced/2009-11-17-combining-repositories/ - -Still to Come -------------- - -At this point I felt the site was ready to be released, so I set up a new site -on WebFaction and used Fabric to deploy it. - -I'm very happy with the result, but there are still a few things I'm going to -fix/change in the future. - -### "Ago" Dates - -**UPDATE:** This is done. I've started using [timeago.js][] to render the -"ago" dates. - -If you look at the top of each blog entry and project there's a line beneath -the title that looks something like: - - :::text - Posted on Monday, November 16, 2009 (1 month, 3 weeks ago). - -The `(1 month, 3 weeks ago)` part of that line is something that I really -appreciate on blogs. When they just list the date I always have to do the math -in my head to figure out roughly how old something is. - -With a static site, however, those times will quickly become inaccurate if I -don't regenerate and publish the site regularly. I'm still thinking about the -best way to work this out. - -One option is to use a cron job on the WebFaction server to regenerate the -site every day, which would keep the times *fairly* accurate. - -Another option would be to use a bit of JavaScript to calculate and render the -time when the page is loaded. This would make it *completely* accurate but -wouldn't work if someone is browsing with JavaScript turned off. - -[timeago.js]: http://timeago.yarp.com/ - -### Categories - -**UPDATE:** I've added categories. Check out [this changeset][categorycset] to see what I had to do. - -Right now all the blog entries (and projects) are listed in a single -chronological list. It would be great to break them up into categories so -people can easily find the articles they're interested in. - -Hyde supports categories but I haven't spent the time to learn how to use them -yet. I also need to figure out a way to work a list of categories into the -design without cluttering things up too much. - -[categorycset]: http://bitbucket.org/sjl/stevelosh/changeset/08d7552b6237/ - -### Use LessCSS - -**UPDATE:** This is done. The main style file now uses LessCSS. - -[LessCSS][] is a language that extends CSS with some useful features like -variables, mixins, operations and nested rules. It can make the styles of a -site much, much cleaner. - -Hyde includes a LessCSS "processor" that will automatically render your -LessCSS files into normal CSS. I'm planning on rewriting the site's styles in -LessCSS and using the processor once I get some free time. - -[lesscss]: {{links.lesscss}} - -View the Code -------------- - -The code is on [BitBucket][bb-repo] and [GitHub][gh-repo]. Feel free to poke -around and see how I've set things up. If you have questions or suggestions -I'd love to hear them! - -[bb-repo]: http://bitbucket.org/sjl/stevelosh/ -[gh-repo]: http://github.com/sjl/stevelosh/ - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2010/01/moving-from-django-to-hyde.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2010/01/moving-from-django-to-hyde.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,462 @@ ++++ +title = "Moving from Django to Hyde" +snip = "Another year, another rewrite." +date = 2010-01-15T20:14:00Z +draft = false + ++++ + +Almost exactly one year ago I posted a blog entry about how I [rewrote this +site][rewrite] using [Django][]. It's a new year, and once again I've +rewritten the site. + +If you've visited my site before you may have noticed some small style tweaks. +I've cleaned up a few things and I think the site looks better overall. The +biggest change, however, is that I completely rewrote the core — this time +with [Hyde][]. + +Hyde is a "static site generator" written in [Python][], similar to +[Jekyll][], [Blatter][] and [Pilcrow][]. I chose it over the others because +it's written in Python (unlike Jekyll), more flexible than Blatter and more +mature than Pilcrow. + +Rewriting an existing site that gets a decent amount of visitors is different +than creating a new site from scratch, so I decided to write this entry to +describe some of the issues I ran into and how I tackled them. + +[Django]: {{links.django}} +[rewrite]: /blog/2009/01/site-redesign/ +[Hyde]: {{links.hyde}} +[Python]: {{links.python}} +[Jekyll]: http://jekyllrb.com/ +[Blatter]: {{links.blatter}} +[Pilcrow]: http://inky.github.com/pilcrow/ + +{{% toc %}} + +Why Static? +----------- + +There are a couple of reasons I decided to switch to a static site. + +### Less Memory Needed + +I use [WebFaction][] for my web hosting, and with my current plan I have a +limit on how much memory I can use at any given time. Each Django site takes +up around 20mb of memory on the server. + +By switching to a static set of HTML pages for my site I save those 20mb so I +can use them for something else (like a staging site for another project). My +site doesn't particularly *need* all of the functionality Django can provide, +so I figured I'd switch to a static site and save the memory. + +[WebFaction]: {{links.webfaction}} + +### Static Pages are Faster + +My site doesn't get an enormous amount of traffic, so the Django instances +weren't really working very hard. Still, serving a static page through Apache +or nginx is always going to be faster than running it through Django. +Faster-loading pages is always a good thing, even if the speed increase isn't +very large. + +### Easier to Maintain + +This is the main reason I switched. Previously, to edit a blog entry I would +log in through Django's admin interface and edit the entry. With Hyde, I +simply edit a file (or create a new one) on my machine and then run a single +command to publish it. + +Another problem with the old site is that I needed to be connected to the +internet to get to the admin interface, so I couldn't update entries (or +publish new ones) offline. I usually have an internet connection, but +occasionally I don't. Now I can edit as much as I like on my own machine and +only need a connection to publish the finished product. + +### Easier to Backup + +One more advantage of Hyde is that the site structure *and* content are all +held in the same place. I keep everything in a [Mercurial][] repository, and +so every time I push that repository somewhere it creates a full backup of the +site's code *and* content. If WebFaction's server catches on fire I still have +everything on my local machine (and on BitBucket). + +I've toyed around with backing up Django's database tables when I had my old +site, but this new method is far less work. + +[Mercurial]: {{links.mercurial}} + +Getting Started +--------------- + +I wanted a fresh start when I was rewriting the site, so I went ahead and +created a brand new folder and Mercurial repository for it. + +I used `hyde --init` to lay out a skeleton in the new folder. Then I stripped +out a lot of the default items that get added with `--init` and created the +directory structure I wanted to use, with stubs for each of the main content +files I knew I'd need. + +Finally, I went ahead and filled some of the basic values in the `settings.py` +file. + +Layout and Styling +------------------ + +Once I had a skeleton in place I started working on layout of the site. + +### Cleanup + +The templates created by `hyde --init` are functional, but when you look at +the code they're a mess. The indentation is strange and inconsistent and +there's trailing whitespace all over the place. I like clean code so I sat +down and cleaned everything up before I started making any real changes. + +### Rewriting the Layout and Styles + +After I finished cleaning up the templates I duplicated the HTML structure and +styles of the old site from scratch. The old site had gone through a bunch of +iterations and I was a bit sloppy in my editing, so there was a lot of cruft +that had snuck in. I wanted a truly *fresh* start for the site, so I buckled +down and did it all again. + +### A New CSS Framework + +Previously I used the [Blueprint CSS framework][blueprint] to make laying out +the site easier. Blueprint is a great framework, but it's more powerful than I +need for a site as simple as this one. The site now uses [Aardvark Legs][aal], +which is a much simpler framework that simply sets up a great vertical rhythm +and leaves you free to lay out the horizontal structure yourself. + +### Pagination + +Before the rewrite the list of blog entries used to be paginated. Hyde +supports pagination but I decided against using it because I simply don't +write enough to make it necessary. All the blog entries are now listed on a +single page, which means that you can use Cmd+F to find an article if you're +looking for something specific. + +[blueprint]: http://www.blueprintcss.org/ +[aal]: {{links.aardvarklegs}} + +Using Fabric to Type Less +------------------------- + +I realized very quickly that typing `hyde -g -s . && hyde -w -s .` would get +old pretty quickly, so I installed [Fabric][] and wrote a fabfile. + +Fabric is a tool written in Python that lets you define tasks and execute them +by running `fab taskname`. Fabfiles are pure Python, so you can build larger +tasks out of smaller ones very easily and do just about anything you want. +It's similar to [ant][], but without the excessive over-engineering. + +You can view the [current fabfile][fabfile] on BitBucket. At the time of this +entry it looks like this: + +```python +from fabric.api import * +import os +import fabric.contrib.project as project + +PROD = 'sjl.webfactional.com' +DEST_PATH = '/home/sjl/webapps/slc/' +ROOT_PATH = os.path.abspath(os.path.dirname(__file__)) +DEPLOY_PATH = os.path.join(ROOT_PATH, 'deploy') + +def clean(): + local('rm -rf ./deploy') + +def regen(): + clean() + local('hyde -g -s .') + +def serve(): + local('hyde -w -s .') + +def reserve(): + regen() + serve() + +def smush(): + local('smusher ./media/images') + +@hosts(PROD) +def publish(): + regen() + project.rsync_project( + remote_dir=DEST_PATH, + local_dir=DEPLOY_PATH.rstrip('/') + '/', + delete=True + ) +``` + +The task I use most often while developing is `fab reserve`, which regenerates +the site and then starts serving it so I can view the result in a browser. + +I use `fab smush` whenever I add new images. This runs [smusher][] on all of +the images to optimize them without reducing quality. + +When I'm ready to publish changes to the live site I run `fab publish`, which +will regenerate my local version and copy it up to the WebFaction server. + +[Fabric]: {{links.fabric}} +[ant]: http://ant.apache.org/ +[fabfile]: http://bitbucket.org/sjl/stevelosh/src/tip/fabfile.py +[smusher]: http://github.com/grosser/smusher + +Moving the Content +------------------ + +The content of the old site (blog entries, projects, and static pages like the +[about page][]) was fairly easy to migrate over to the new one, because both +versions use [Markdown][] to format the text. + +First I created an empty file for each page with a filename that matched the +"slug" (last part of the URL) of the old page. Then I manually copied over the +title, creation time and content for every page. I could have written a script +to do this, but I don't have enough pages on the site to make it worth the +time. + +[about page]: /about/ +[Markdown]: {{links.markdown}} + +Converting the Comments +----------------------- + +At this point the new site was looking very much like the old one. The styles +were (roughly) matching and the blog posts, entries, and static pages were all +rendered nicely. + +The next big task was migrating all of the comments. Since the new site is +static it can't handle dynamic content like comments on its own, so I decided +to use [Disqus][]. I use Disqus for the comments on the [hg tip][] site and +it's very nice. For that site I used it from the beginning, but for this +rewrite I needed to somehow import all the old comments. + +To migrate the comments over I used [django-disqus][], but there was a small +snag that I needed to deal with first. + +When I first wrote the old site I was just getting back into Django after not +using it for a long time. I didn't know about Django's built-in comment +models, so I created my own. They weren't as good as Django's, but they did +the job and I didn't care enough to change them. + +This became a problem when it was time to import the old comments into Disqus. +django-disqus only supports importing comments from Django's built-in comment +models. To work around this I first had to convert the old comments into +Django's built-in ones. I wrote a [small, hacky Python +script][convert-comments] to do it: + + :::python + #!/usr/bin/env python + + from django.core.management import setup_environ + import settings + setup_environ(settings) + + from markdown import Markdown + from django.contrib.comments.models import Comment + from django.contrib.sites.models import Site + from stevelosh.blog.models import Comment as BlogComment + from stevelosh.projects.models import Comment as ProjectComment + + + mdown = Markdown() + + site = Site.objects.all()[0] + blog_comments = BlogComment.objects.filter(spam=False) + project_comments = ProjectComment.objects.filter(spam=False) + + for bc in blog_comments: + c = Comment() + c.content_object = bc.entry + c.user_name = bc.name + c.comment = mdown.convert(bc.body) + c.submit_date = bc.submitted + c.site = site + c.is_public = True + c.is_removed = False + c.save() + # print 'http://%s%s' % (site.domain, c.content_object.get_absolute_url()) + + for pc in project_comments: + c = Comment() + c.content_object = pc.project + c.user_name = pc.name + c.comment = mdown.convert(pc.body) + c.submit_date = pc.submitted + c.site = site + c.is_public = True + c.is_removed = False + c.save() + # print 'http://%s%s' % (site.domain, c.content_object.get_absolute_url()) + +Yes, it's ugly. No, I don't care. It was run once or twice locally and once on +the live server. It worked and I'll never need to run it again. + +Once I had the comments converted I could use django-disqus to migrate them. +The import went very smoothly — I ran one command and after a couple of +minutes everything was in Disqus. Once that finished it was just a matter of +adding the Disqus JavaScript to one of the templates. + +[Disqus]: http://disqus.com/ +[hg tip]: {{links.hgtip}} +[django-disqus]: http://github.com/arthurk/django-disqus +[convert-comments]: http://bitbucket.org/sjl/stevelosh/src/da98105753a1/convert-comments.py + +Rewriting the Old URLs +---------------------- + +Since I was pretty much starting from scratch with this rewrite I decided to +clean up the URL structure of my blog. Previously a blog entry's URL looked +like this: + +```text +http://stevelosh.com/blog/entry/2009/08/30/a-guide-to-branching-in-mercurial/ +``` + +The `/entry/` part of the URL was useless — it just took up space, so I got +rid of it. The year and month are useful to get an idea of how old an entry is +just by looking at the link, so I left them in. The day, however, probably +doesn't matter that much, so I took it out. + +The new URLs look like this: + +```text +http://stevelosh.com/blog/2009/08/a-guide-to-branching-in-mercurial/ +``` + +The problem with rewriting the URL structure is that there are already links +around the web pointing at my entries. I didn't want those old links to break, +so I crafted an [`.htaccess` file][htaccess] that would rewrite the old URLs +into the new ones: + + {% templatetag openblock %} if GENERATE_CLEAN_URLS {% templatetag closeblock %} + RewriteEngine on + RewriteBase {% templatetag openvariable %} node.site.settings.SITE_ROOT {% templatetag closevariable %} + + # Old URLs + RewriteRule ^blog/entry/(\d+)/(\d\d)/\d+/([^/]*)/?$ /blog/$1/$2/$3/ [R=301,L] + RewriteRule ^blog/entry/(\d+)/(\d)/\d+/([^/]*)/?$ /blog/$1/0$2/$3/ [R=301,L] + + {% templatetag openblock %} hyde_listing_page_rewrite_rules {% templatetag closeblock %} + + # listing pages whose names are the same as their enclosing folder's + RewriteCond %{REQUEST_FILENAME}/$1.html -f + RewriteRule ^([^/]*)/$ %{REQUEST_FILENAME}/$1.html + + # regular pages + RewriteCond %{REQUEST_FILENAME}.html -f + RewriteRule ^.*$ %{REQUEST_FILENAME}.html + + {% templatetag openblock %} endif {% templatetag closeblock %} + +Most of that file is the stock Hyde `.htaccess` file — the two lines under `# +Old URLs` are the ones that I added. It took me a while to get it right +because I don't work with Apache's `mod_rewrite` very often, but it was worth +it to avoid breaking all of the old links. + +[htaccess]: http://bitbucket.org/sjl/stevelosh/src/tip/content/.htaccess + +Creating an RSS Feed +-------------------- + +I had an RSS feed for the old site which Django made very easy to set up. I +definitely needed a feed for the new site, and fortunately Hyde provides a +simple sample template that can be used to make an ATOM feed. + +I cleaned up the whitespace and formatting of that template a bit, adjusted a +few variables for my needs, put it on [FeedBurner][] and everything was ready. + +[FeedBurner]: http://www.feedburner.com/ + +Merging the Repositories +------------------------ + +The last step before I was finished was to merge the old and new repositories +together so I wouldn't lose any of the history. It's probably not *too* +important to keep the old site's history around, but I put a lot of work into +it over the past year and it has some sentimental value. + +Fortunately it's very easy to [combine Mercurial repositories][combinerepos]. +I just pulled the old repository into the new one, merged the old head into +the new one while discarding all the changes, and pushed it up to BitBucket. + +[combinerepos]: http://hgtip.com/tips/advanced/2009-11-17-combining-repositories/ + +Still to Come +------------- + +At this point I felt the site was ready to be released, so I set up a new site +on WebFaction and used Fabric to deploy it. + +I'm very happy with the result, but there are still a few things I'm going to +fix/change in the future. + +### "Ago" Dates + +**UPDATE:** This is done. I've started using [timeago.js][] to render the +"ago" dates. + +If you look at the top of each blog entry and project there's a line beneath +the title that looks something like: + +```text +Posted on Monday, November 16, 2009 (1 month, 3 weeks ago). +``` + +The `(1 month, 3 weeks ago)` part of that line is something that I really +appreciate on blogs. When they just list the date I always have to do the math +in my head to figure out roughly how old something is. + +With a static site, however, those times will quickly become inaccurate if I +don't regenerate and publish the site regularly. I'm still thinking about the +best way to work this out. + +One option is to use a cron job on the WebFaction server to regenerate the +site every day, which would keep the times *fairly* accurate. + +Another option would be to use a bit of JavaScript to calculate and render the +time when the page is loaded. This would make it *completely* accurate but +wouldn't work if someone is browsing with JavaScript turned off. + +[timeago.js]: http://timeago.yarp.com/ + +### Categories + +**UPDATE:** I've added categories. Check out [this changeset][categorycset] to see what I had to do. + +Right now all the blog entries (and projects) are listed in a single +chronological list. It would be great to break them up into categories so +people can easily find the articles they're interested in. + +Hyde supports categories but I haven't spent the time to learn how to use them +yet. I also need to figure out a way to work a list of categories into the +design without cluttering things up too much. + +[categorycset]: http://bitbucket.org/sjl/stevelosh/changeset/08d7552b6237/ + +### Use LessCSS + +**UPDATE:** This is done. The main style file now uses LessCSS. + +[LessCSS][] is a language that extends CSS with some useful features like +variables, mixins, operations and nested rules. It can make the styles of a +site much, much cleaner. + +Hyde includes a LessCSS "processor" that will automatically render your +LessCSS files into normal CSS. I'm planning on rewriting the site's styles in +LessCSS and using the processor once I get some free time. + +[lesscss]: {{links.lesscss}} + +View the Code +------------- + +The code is on [BitBucket][bb-repo] and [GitHub][gh-repo]. Feel free to poke +around and see how I've set things up. If you have questions or suggestions +I'd love to hear them! + +[bb-repo]: http://bitbucket.org/sjl/stevelosh/ +[gh-repo]: http://github.com/sjl/stevelosh/ + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2010/01/the-real-difference-between-mercurial-and-git.html --- a/content/blog/2010/01/the-real-difference-between-mercurial-and-git.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,136 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "The Real Difference Between Mercurial and Git" - snip: "It’s not their features." - created: 2010-01-20 21:56:00 -%} - -{% block article %} - -There are a -[lot](http://nubyonrails.com/articles/five-features-from-mercurial-that-would-make-git-suck-less) -of -[other](http://www.rockstarprogrammer.org/post/2008/apr/06/differences-between-mercurial-and-git/) -[blog](http://importantshock.wordpress.com/2008/08/07/git-vs-mercurial/) -[posts](http://jointheconversation.org/2008/11/24/on-mercurial.html) -and -[web sites](http://www.whygitisbetterthanx.com/) -comparing [Mercurial][] and [git][]. - -Some of them are just plain outdated or wrong -([whygitisbetterthanx.com](http://whygitisbetterthanx.com/) for example, lists -"cheap local branching" as a git advantage over Mercurial, when Mercurial -actually has [*cheaper* branching][hgbranch] than git!). - -Even the ones that aren't factually incorrect often seem to focus on features -and speed. While this may have been relevant in the past, both systems have -borrowed so much from each other that I don't think it's a very big deal -today. - -This post is about what I feel are the last, most *important* differences -between git and Mercurial. - -[Mercurial]: {{links.mercurial}} -[git]: {{links.git}} -[hgbranch]: /blog/2009/08/a-guide-to-branching-in-mercurial/#branching-anonymously - -[TOC] - -The Big Difference ------------------- - -I think there's still one very, very important difference left: **the systems -*feel* very different to use**. - -This might sound trivial, but I don't think it is. How a system "feels" -matters. Version control is something you use *constantly*, so it's important -to find a system that fits your way of thinking. - -I'm a visual learner, so I find it helpful to make visual analogies when I'm -trying to explain something. Here's how I visualize the difference between -Mercurial and git: - -![Swiss Army Knife and Kitchen Utensils](/media/images{{parent_url}}/mercurial-vs-git.jpg "Mercurial vs. Git") - -Each git command is like a Swiss Army knife. For example, `git checkout` can -switch the working directory to a new branch, update file contents to that of -a previous revision, and even create a new branch. It's an efficient way to -work once you learn all the arguments and how they interact with each other. - -Mercurial is like a well-equipped kitchen -- it has a lot of tools that each -do one simple, well-defined thing, and do it well. To switch the working -directory you use `hg update`. To update file contents to what they were at a -previous revision you use `hg revert`. To create a new branch you use `hg -branch`. This means there are more commands to learn, but each command is much -simpler and more specific to a single conceptual task. - -I personally find Mercurial's philosophy easier to work with. It's easy for me -to wrap my head around commands when they're more modular and less monolithic. - -For example, I find Mercurial's `help` entries easier to understand. This -might just be because git's `help` entries are simply too long to read and -digest when I'm in the middle of coding: - - :::console - $ git help checkout | wc -l - 236 - - $ (hg help update && hg help branch && hg help revert) | wc -l - 100 - -I realize that not everyone feels the same way as I do. If you like git's way -of thinking better, you should certainly use it. - -Other Differences ------------------ - -Aside from their "feels" I see two other big differences that still remain -between git and Mercurial. - -### BitBucket and GitHub - -A major difference between the two systems is their most popular hosting -sites: [BitBucket][] and [GitHub][]. GitHub is leaps and bounds above -BitBucket in pretty much any aspect you look at: graphic design, popularity, -speed, and/or features. Every time I've seriously considered moving to git it -was purely because of this disparity. - -I don't like using git itself (though it's *far* better than SVN or CVS), but -GitHub is such an awesome site that I've considered switching just to use it. -You can chalk it up to "popularity" (i.e. git has more users, so GitHub is -more profitable and so has more money to spend on development), but the fact -is that GitHub is still far better than BitBucket. - -[BitBucket]: {{links.bitbucket}} -[GitHub]: {{links.github}} - -### Git's Index - -One of the features that git users talk about often is git's -[index][gitindex]. Mercurial has the [record][] extension, but it's not really -the same thing. - -I personally don't like the index. I feel that git encourages people to check -in changesets that contain code which they've never tested (or even built) -because the index is such a prominent part of git's workflow. - -Yes, one can argue that you don't actually *push* changes until you've got a -working state. The problem is that when you then try to run `git bisect` later -to find a bug you'll have to waste time skipping over changesets that don't -build. You could use `git rebase --interactive` to fold partial changesets -into one big one, but again, my gut feeling is that few people actually bother -*testing* the results. - -With all that said, I've [considered][] creating a Mercurial extension that -adds git's index functionality to Mercurial, because I think it would make the -transition to Mercurial easier for git users. I've had enough experience with -Mercurial's internals to know that such a thing is *possible*, but I simply -don't have the time to do it *all* myself. If you're interested in helping out -please let me know! - -[gitindex]: http://book.git-scm.com/1_the_git_index.html -[record]: http://mercurial.selenic.com/wiki/RecordExtension -[considered]: http://twitter.com/stevelosh/status/7106305374 - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2010/01/the-real-difference-between-mercurial-and-git.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2010/01/the-real-difference-between-mercurial-and-git.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,134 @@ ++++ +title = "The Real Difference Between Mercurial and Git" +snip = "It’s not their features." +date = 2010-01-20T21:56:00Z +draft = false + ++++ + +There are a +[lot](http://nubyonrails.com/articles/five-features-from-mercurial-that-would-make-git-suck-less) +of +[other](http://www.rockstarprogrammer.org/post/2008/apr/06/differences-between-mercurial-and-git/) +[blog](http://importantshock.wordpress.com/2008/08/07/git-vs-mercurial/) +[posts](http://jointheconversation.org/2008/11/24/on-mercurial.html) +and +[web sites](http://www.whygitisbetterthanx.com/) +comparing [Mercurial][] and [git][]. + +Some of them are just plain outdated or wrong +([whygitisbetterthanx.com](http://whygitisbetterthanx.com/) for example, lists +"cheap local branching" as a git advantage over Mercurial, when Mercurial +actually has [*cheaper* branching][hgbranch] than git!). + +Even the ones that aren't factually incorrect often seem to focus on features +and speed. While this may have been relevant in the past, both systems have +borrowed so much from each other that I don't think it's a very big deal +today. + +This post is about what I feel are the last, most *important* differences +between git and Mercurial. + +[Mercurial]: {{links.mercurial}} +[git]: {{links.git}} +[hgbranch]: /blog/2009/08/a-guide-to-branching-in-mercurial/#branching-anonymously + +{{% toc %}} + +The Big Difference +------------------ + +I think there's still one very, very important difference left: **the systems +*feel* very different to use**. + +This might sound trivial, but I don't think it is. How a system "feels" +matters. Version control is something you use *constantly*, so it's important +to find a system that fits your way of thinking. + +I'm a visual learner, so I find it helpful to make visual analogies when I'm +trying to explain something. Here's how I visualize the difference between +Mercurial and git: + +![Swiss Army Knife and Kitchen Utensils](/media/images/blog/2010/01/mercurial-vs-git.jpg "Mercurial vs. Git") + +Each git command is like a Swiss Army knife. For example, `git checkout` can +switch the working directory to a new branch, update file contents to that of +a previous revision, and even create a new branch. It's an efficient way to +work once you learn all the arguments and how they interact with each other. + +Mercurial is like a well-equipped kitchen — it has a lot of tools that each +do one simple, well-defined thing, and do it well. To switch the working +directory you use `hg update`. To update file contents to what they were at a +previous revision you use `hg revert`. To create a new branch you use `hg +branch`. This means there are more commands to learn, but each command is much +simpler and more specific to a single conceptual task. + +I personally find Mercurial's philosophy easier to work with. It's easy for me +to wrap my head around commands when they're more modular and less monolithic. + +For example, I find Mercurial's `help` entries easier to understand. This +might just be because git's `help` entries are simply too long to read and +digest when I'm in the middle of coding: + +```console +$ git help checkout | wc -l +236 + +$ (hg help update && hg help branch && hg help revert) | wc -l +100 +``` + +I realize that not everyone feels the same way as I do. If you like git's way +of thinking better, you should certainly use it. + +Other Differences +----------------- + +Aside from their "feels" I see two other big differences that still remain +between git and Mercurial. + +### BitBucket and GitHub + +A major difference between the two systems is their most popular hosting +sites: [BitBucket][] and [GitHub][]. GitHub is leaps and bounds above +BitBucket in pretty much any aspect you look at: graphic design, popularity, +speed, and/or features. Every time I've seriously considered moving to git it +was purely because of this disparity. + +I don't like using git itself (though it's *far* better than SVN or CVS), but +GitHub is such an awesome site that I've considered switching just to use it. +You can chalk it up to "popularity" (i.e. git has more users, so GitHub is +more profitable and so has more money to spend on development), but the fact +is that GitHub is still far better than BitBucket. + +[BitBucket]: {{links.bitbucket}} +[GitHub]: {{links.github}} + +### Git's Index + +One of the features that git users talk about often is git's +[index][gitindex]. Mercurial has the [record][] extension, but it's not really +the same thing. + +I personally don't like the index. I feel that git encourages people to check +in changesets that contain code which they've never tested (or even built) +because the index is such a prominent part of git's workflow. + +Yes, one can argue that you don't actually *push* changes until you've got a +working state. The problem is that when you then try to run `git bisect` later +to find a bug you'll have to waste time skipping over changesets that don't +build. You could use `git rebase --interactive` to fold partial changesets +into one big one, but again, my gut feeling is that few people actually bother +*testing* the results. + +With all that said, I've [considered][] creating a Mercurial extension that +adds git's index functionality to Mercurial, because I think it would make the +transition to Mercurial easier for git users. I've had enough experience with +Mercurial's internals to know that such a thing is *possible*, but I simply +don't have the time to do it *all* myself. If you're interested in helping out +please let me know! + +[gitindex]: http://book.git-scm.com/1_the_git_index.html +[record]: http://mercurial.selenic.com/wiki/RecordExtension +[considered]: http://twitter.com/stevelosh/status/7106305374 + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2010/02/mercurial-workflows-branch-as-needed.html --- a/content/blog/2010/02/mercurial-workflows-branch-as-needed.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,148 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "Mercurial Workflows: Branch As Needed" - snip: "Part 1 of several." - created: 2010-02-28 14:00:00 -%} - -{% block article_class %}with-diagrams{% endblock %} - -{% block article %} - -A while ago [Vincent Driessen][] posted an example of [a successful git -branching model][gitbranch]. A lot of git users found that article very -helpful, and [Dirkjan Ochtman][] [told me][djctweet] I should write a similar -article for Mercurial users. - -[Vincent Driessen]: http://nvie.com/ -[gitbranch]: http://nvie.com/git-model -[Dirkjan Ochtman]: http://dirkjan.ochtman.nl/ -[djctweet]: http://twitter.com/djco/status/8061889499 - -I decided that I didn't want to just write a single entry about one branching -workflow. Mercurial is flexible enough to support many different workflows and -some of them will fit a given project better than others. Instead I'm going to -write a series of posts, each one about a particular workflow. - -I'm going to start off with the simplest example I can think of: "Don't worry -about branching at all, just deal with it whenever it happens." - -**Note:** In this series I'm going to assume you're comfortable with basic -Mercurial commands and you know how Mercurial's various forms of branching -work. If you need some review on Mercurial's commands you should look at the -[hg book][]. If you need more information on branching concepts you might like -my [Guide to Branching in Mercurial][hgbranching]. - -[hg book]: http://hgbook.red-bean.com/ -[hgbranching]: /blog/2009/08/a-guide-to-branching-in-mercurial/ - -[TOC] - -"Branch as Needed" in a Nutshell --------------------------------- - -The general idea of this workflow is that you don't worry about branching -until it actually happens. - -The benefit is that it takes no extra work up front and keeps things very -simple. - -The drawback is that it doesn't scale very well. It's great for small projects -but it for larger ones you'll probably want something a bit more structured. - -An Example Scenario -------------------- - -This workflow is most suited to small projects. Here's a sample repository -with only a single, linear series of changes: - -![Sample Repository Diagram](/media/images{{ parent_url }}/hg-branching-1-start.png "Sample Repository") - -In this example there's mostly just a single developer (you) working on the -project to add features, fix bugs, etc. - -The repository is online so other people can get the code. They can add -features and fix bugs if they want, but it doesn't happen very often because -it's a small project. - -**Note:** I find it helpful to have graphs of the changesets in a repository -so I can see what's going on. If you want some nice, quick ASCII-art graphs of -your own repositories you can [use the graphlog extension][glog]. - -[glog]: http://hgtip.com/tips/beginner/2009-10-03-stay-sane-with-graphlog/ - -Branch Setup ------------- - -In this workflow you don't do any up-front setup at all. Just use Mercurial as -you normally would, committing your changes along the way and pushing them to -somewhere where other people can get them (like [BitBucket][]). - -[BitBucket]: http://bitbucket.org/ - -Contributing to the Project ---------------------------- - -Let's say someone else starts using your project and finds a bug. They go -ahead and fix the bug themselves and commit a changeset for the fix. They -could then push their copy of the repository to somewhere public (like their -own BitBucket account) and it would look something like this: - -![Contributor Repository Diagram](/media/images{{ parent_url }}/hg-branching-1-other.png "Contributor Repository") - -Once their changes are somewhere public they can email you and say: - -> Hey, I fixed a bug in your project. -> -> The fix is changeset be3063198fea in my copy of your repository at -> http://.../ - -Merging Changes from Contributors ---------------------------------- - -When you get an email like this you would head over to their repository and -take a look at the changeset. If you decide it's good and want to incorporate -it into your own repository it's as simple as running `hg pull -http://their/repo/`. - -If you haven't made any new changes your repository would now look exactly -like theirs. You can update to the new tip and continue working as usual. - -What if you've made changes between the time they cloned (or last pulled) your -repository and the time you read the email & pulled their changes? In that -case your repository will look like this after you pull from them: - -![Sample Repository Before Merging Diagram](/media/images{{ parent_url }}/hg-branching-1-needs-merge.png "Sample Repository Before Merging") - -Because John's bugfix changeset and your refactoring changeset both have the -same parent there are now two "anonymous branches" in your repository. This -doesn't bother Mercurial at all -- repositories can have as many "anonymous -branches" as you like. - -You'll probably want to merge these branches, so you'd run `hg update -a2125cb20c54` (if you weren't already there) and then `hg merge be3063198fea` -to merge John's bugfix with your new changes. The result would look like this: - -![Sample Repository After Merging Diagram](/media/images{{ parent_url }}/hg-branching-1-after-merge.png "Sample Repository After Merging") - -Now you're back to having just one head and you can continue working as usual, -with John's changes and your changes all merged together. - -Summary -------- - -This workflow is the simplest one possible. There's no up-front setup and it's -very easy for new people to contribute to the project -- they just clone, -commit, push, and tell you about their changes. It's great for small projects -with one main developer and the occasional contributor. - -If you have a project with a lot of people working together this can get -pretty chaotic. Your repository graph will end up looking like a tangled mess. -In that case you'll want a workflow with a bit more structure. - -I'm planning on writing at least two or three more posts about some more -complicated branching workflows in the future. If you have any specific -examples you think I should write about please let me know! - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2010/02/mercurial-workflows-branch-as-needed.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2010/02/mercurial-workflows-branch-as-needed.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,143 @@ ++++ +title = "Mercurial Workflows: Branch As Needed" +snip = "Part 1 of several." +date = 2010-02-28T14:00:00Z +draft = false + ++++ + +A while ago [Vincent Driessen][] posted an example of [a successful git +branching model][gitbranch]. A lot of git users found that article very +helpful, and [Dirkjan Ochtman][] [told me][djctweet] I should write a similar +article for Mercurial users. + +[Vincent Driessen]: http://nvie.com/ +[gitbranch]: http://nvie.com/git-model +[Dirkjan Ochtman]: http://dirkjan.ochtman.nl/ +[djctweet]: http://twitter.com/djco/status/8061889499 + +I decided that I didn't want to just write a single entry about one branching +workflow. Mercurial is flexible enough to support many different workflows and +some of them will fit a given project better than others. Instead I'm going to +write a series of posts, each one about a particular workflow. + +I'm going to start off with the simplest example I can think of: "Don't worry +about branching at all, just deal with it whenever it happens." + +**Note:** In this series I'm going to assume you're comfortable with basic +Mercurial commands and you know how Mercurial's various forms of branching +work. If you need some review on Mercurial's commands you should look at the +[hg book][]. If you need more information on branching concepts you might like +my [Guide to Branching in Mercurial][hgbranching]. + +[hg book]: http://hgbook.red-bean.com/ +[hgbranching]: /blog/2009/08/a-guide-to-branching-in-mercurial/ + +{{% toc %}} + +"Branch as Needed" in a Nutshell +-------------------------------- + +The general idea of this workflow is that you don't worry about branching +until it actually happens. + +The benefit is that it takes no extra work up front and keeps things very +simple. + +The drawback is that it doesn't scale very well. It's great for small projects +but it for larger ones you'll probably want something a bit more structured. + +An Example Scenario +------------------- + +This workflow is most suited to small projects. Here's a sample repository +with only a single, linear series of changes: + +Sample Repository + +In this example there's mostly just a single developer (you) working on the +project to add features, fix bugs, etc. + +The repository is online so other people can get the code. They can add +features and fix bugs if they want, but it doesn't happen very often because +it's a small project. + +**Note:** I find it helpful to have graphs of the changesets in a repository +so I can see what's going on. If you want some nice, quick ASCII-art graphs of +your own repositories you can [use the graphlog extension][glog]. + +[glog]: http://hgtip.com/tips/beginner/2009-10-03-stay-sane-with-graphlog/ + +Branch Setup +------------ + +In this workflow you don't do any up-front setup at all. Just use Mercurial as +you normally would, committing your changes along the way and pushing them to +somewhere where other people can get them (like [BitBucket][]). + +[BitBucket]: http://bitbucket.org/ + +Contributing to the Project +--------------------------- + +Let's say someone else starts using your project and finds a bug. They go +ahead and fix the bug themselves and commit a changeset for the fix. They +could then push their copy of the repository to somewhere public (like their +own BitBucket account) and it would look something like this: + +Contributor Repository + +Once their changes are somewhere public they can email you and say: + +> Hey, I fixed a bug in your project. +> +> The fix is changeset be3063198fea in my copy of your repository at +> http://.../ + +Merging Changes from Contributors +--------------------------------- + +When you get an email like this you would head over to their repository and +take a look at the changeset. If you decide it's good and want to incorporate +it into your own repository it's as simple as running `hg pull +http://their/repo/`. + +If you haven't made any new changes your repository would now look exactly +like theirs. You can update to the new tip and continue working as usual. + +What if you've made changes between the time they cloned (or last pulled) your +repository and the time you read the email & pulled their changes? In that +case your repository will look like this after you pull from them: + +Sample Repository Before Merging + +Because John's bugfix changeset and your refactoring changeset both have the +same parent there are now two "anonymous branches" in your repository. This +doesn't bother Mercurial at all — repositories can have as many "anonymous +branches" as you like. + +You'll probably want to merge these branches, so you'd run `hg update +a2125cb20c54` (if you weren't already there) and then `hg merge be3063198fea` +to merge John's bugfix with your new changes. The result would look like this: + +Sample Repository After Merging + +Now you're back to having just one head and you can continue working as usual, +with John's changes and your changes all merged together. + +Summary +------- + +This workflow is the simplest one possible. There's no up-front setup and it's +very easy for new people to contribute to the project — they just clone, +commit, push, and tell you about their changes. It's great for small projects +with one main developer and the occasional contributor. + +If you have a project with a lot of people working together this can get +pretty chaotic. Your repository graph will end up looking like a tangled mess. +In that case you'll want a workflow with a bit more structure. + +I'm planning on writing at least two or three more posts about some more +complicated branching workflows in the future. If you have any specific +examples you think I should write about please let me know! + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2010/02/my-extravagant-zsh-prompt.html --- a/content/blog/2010/02/my-extravagant-zsh-prompt.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,485 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "My Extravagant Zsh Prompt" - snip: "It’s big, but my monitor isn’t running out of ink." - created: 2010-02-01 01:05:00 -%} - -{% block article %} - -I spend a lot of time in a Terminal window at a command line. Up until about a -month ago I was using [bash][] for my shell. I decided to try switching to -[Zsh][] after hearing a lot of good things about it and I'm very happy with -the change. - -A few days ago I [tweeted][] my current Zsh prompt and the general response -was: "Cool, but how did you do it?" I promised to write more about it when I -got some free time, and it looks like that time is now. - -**One quick note:** This entry is about the prompt that *I* find useful. You -are not me, so you'll almost certainly have different needs. That's great! -Take this prompt and hack it to make whatever works for you. If you're feeling -generous you should find me on [Twitter][twsl] and show me what you changed -- -I might want to steal/appropriate your changes. - -**Another quick note:** I've customized the colors of my Terminal. They're -based on the [Monokai][] TextMate theme, and I think they look very nice. The -colors will be different for you. If you want colors like mine you should take -a look at the [entry][candy] I wrote about it. - -[bash]: http://en.wikipedia.org/wiki/Bash -[Zsh]: http://www.zsh.org/ -[tweeted]: http://twitter.com/stevelosh/status/8259755151 -[Monokai]: http://www.monokai.nl/blog/2006/07/15/textmate-color-theme/ -[candy]: /blog/2009/03/candy-colored-terminal/ -[twsl]: {{links.twsl}} - -[TOC] - -Why Should You Care? --------------------- - -Many people use the command line every day and never bother to customize their -prompts. It's just a bit of text that's printed before every command -- why -should you waste time learning how to customize it? - -I feel that the *most* important aspect of my command line work is the prompt. -Your prompt is something you'll see *literally* thousands of times a day. Why -not take 30 minutes and customize it into something that's much more useful? - -I firmly believe I'm right in thinking this way. 30 minutes (or even 3 hours) -of customization to make your work easier for the **rest of your life** (or at -least however long you stay with your current shell) *is* worth it. - -This Entry is About Zsh Prompts -------------------------------- - -As I mentioned earlier I now use Zsh as my command line shell. If you use bash -(the default on most modern systems) the syntax to create the shell will be -different. - -Really, though, you should give Zsh a try. - -Screenshots ------------ - -I can write all day, but in the end I think a screenshot will be more helpful -than anything I write. - -Here's a sample of my current Zsh prompt: - -![My Zsh Prompt](/media/images{{parent_url}}/zsh-prompt.png "My Zsh Prompt") - -And here's a version of that screenshot with some comments added to explain -things: - -![My Zsh Prompt with Comments](/media/images{{parent_url}}/zsh-prompt-comments.png "My Zsh Prompt with Comments") - -If you want to know *how* I created that prompt, read on! - -Oh My Zsh! ----------- - -The first thing I'd do when starting out with Zsh is to install -[robbyrussell's][robr] [oh-my-zsh][omz]. It's a great collection of very -useful Zsh configurations and aliases which set some sane defaults and make -working with Zsh much nicer. - -The instructions for installing it are on the project page. - -[robr]: http://github.com/robbyrussell -[omz]: http://github.com/robbyrussell/oh-my-zsh - -Define a "Theme" ----------------- - -oh-my-zsh uses [theme files][themes] to specify how your prompt looks. - -An oh-my-zsh theme file is just a Zsh script which sets a few variables that -the oh-my-zsh scripts use to render your prompt. - -Go ahead and create a new theme file for your prompt. Call it whatever you -like. I called my theme "prose" because it's more verbose than the others. -Copy the contents of one of the other themes to get started. - -The `PROMPT` variable is what you'll be modifying to change how your prompt -looks. That's the `PROMPT=whatever` line of the theme file. - -Once you're done making your awesome prompt you should fork oh-my-zsh on -GitHub and commit/push your new theme so other people can see and use your -prompt. - -[themes]: http://github.com/robbyrussell/oh-my-zsh/tree/master/themes/ - -Username and Hostname ---------------------- - -The first two pieces of my prompt are the simplest: username and hostname. I -SSH between machines pretty frequently so I find it nice to have these in my -prompt to remind me of where I am. - -These are things that you'll find in many, many Zsh prompts. For more -information about this kind of stuff check out [this page][zshpr]. - -After adding in the colors this piece of my `PROMPT` variable looks like this: - - :::text - %{$fg[magenta]%}%n%{$reset_color%} at %{$fg[yellow]%}%m%{$reset_color%} - -[zshpr]: http://www.acm.uiuc.edu/workshops/zsh/prompt/escapes.html - -Current Directory ------------------ - -I like to have the current working directory displayed in my prompt. Zsh has -two built-in ways to show the current directory: - -* **%d** shows the *entire* path. -* **%~** shows the path with any variables replaced. - -Unfortunately neither of these work for me. - -If I'm in a directory that's under my home directory I want to see `~` instead -of the full path. This would seem to mean that I should use `%~`, but there's -a catch. - -The `%~` sequence also replaces any directories that happen to be environment -variables. That means if I'm in something under my `virtualenvs` directory -I'll see `~WORKON_HOME/venv_name` which is extremely ugly. - -To get around this I defined a simple function that implements the behavior I -want. - -To add this to your prompt you'll first need to add the following function to -your theme: - - :::bash - function collapse_pwd { - echo $(pwd | sed -e "s,^$HOME,~,") - } - -Then add `$(collapse_pwd)` to your `PROMPT` variable wherever you'd like the -directory to show up. - -After adding in the colors this piece of my `PROMPT` variable looks like this: - - :::text - in %{$fg_bold[green]%}$(collapse_pwd)%{$reset_color%} - -**UPDATE:** In a comment Juliano pointed out a better way to do this. I've -gotten rid of the `collapse_pwd` function and put this into my `PROMPT` -variable instead: - - :::bash - ${PWD/#$HOME/~} - -My Right-Prompt: Battery Capacity ---------------------------------- - -The next piece of my prompt I'll talk about is the most stand-alone one: the -battery charge. - -I work exclusively on laptops. I don't own a desktop machine -- I have a -Macbook, Macbook Pro and Asus EEE PC and my work machine is a Macbook Pro. I -work from coffeeshops and client meetings pretty often, so it's nice to have a -reminder of my remaining battery power to know when I need to plug in. - -On the right-hand side of the prompt in the screenshot you can see a series of -green triangles. These represent my current battery charge and they turn to -empty triangles as the battery becomes depleted. Once the battery reaches 60% -they turn yellow and once it reaches 40% they turn red. - -To do this I wrote a *very* simple Python script to output my current battery -capacity. The entire script is shown below. - - :::python - #!/usr/bin/env python - # coding=UTF-8 - - import math, subprocess - - p = subprocess.Popen(["ioreg", "-rc", "AppleSmartBattery"], stdout=subprocess.PIPE) - output = p.communicate()[0] - - o_max = [l for l in output.splitlines() if 'MaxCapacity' in l][0] - o_cur = [l for l in output.splitlines() if 'CurrentCapacity' in l][0] - - b_max = float(o_max.rpartition('=')[-1].strip()) - b_cur = float(o_cur.rpartition('=')[-1].strip()) - - charge = b_cur / b_max - charge_threshold = int(math.ceil(10 * charge)) - - # Output - - total_slots, slots = 10, [] - filled = int(math.ceil(charge_threshold * (total_slots / 10.0))) * u'▸' - empty = (total_slots - len(filled)) * u'▹' - - out = (filled + empty).encode('utf-8') - import sys - - color_green = '%{%}' - color_yellow = '%{%}' - color_red = '%{%}' - color_reset = '%{%}' - color_out = ( - color_green if len(filled) > 6 - else color_yellow if len(filled) > 4 - else color_red - ) - - out = color_out + out + color_reset - sys.stdout.write(out) - -**Note:** The `color_*` variables contain UTF-8 characters which don't show up -in a normal web browser. They *should* be preserved when copying and pasting -from a decent browser. If the colors don't work find me on [Twitter][twsl] and -I'll put the raw script up somewhere so you can download it. - -**UPDATE:** [ehamberg][] [sent me][ehbtweet] the beginnings of a -Linux-compatible battery capacity script. If you flesh it out and make it into -a complete script please let me know and I'll add it here. - -[ehamberg]: http://twitter.com/ehamberg/ -[ehbtweet]: http://twitter.com/ehamberg/status/8503213690 - -I've put this script in a file named `batcharge.py` in my `~/bin/` directory. -You can of course name it and place it anything/anywhere you like. - -No, it's not perfect. No, I don't care. It gets the job done and any -brainpower I could spend on making it more efficient could be better spent on -writing or working on something else. - -If you want to improve it, feel free and *please* send me the link to your work on -[Twitter][twsl] so I can use it! - -Now that you've got the script, it's time to add it to your prompt. - -If you want this to appear on the right-hand side of your screen (like mine -does) you'll need to add two things to your theme file. The first is a -function that calls the script: - - :::bash - function battery_charge { - echo `$BAT_CHARGE` 2>/dev/null - } - -After that you'll need to define the Zsh `RPROMPT` variable: - - :::bash - RPROMPT='$(battery_charge)' - -Once those are in your theme file any new Terminal windows you open should -show the battery capacity to the right of your prompt! - -Repository Types ----------------- - -In my personal work and my full-time job I work with both -[Mercurial][mercurial] and [git][] repositories. I find it helpful to have a -visual reminder of what kind of repository I'm working in. - -To do this I've replaced the standard `$` character in my prompt with -different Unicode characters depending on what kind of repository I'm -currently in: - -* `○` means "I'm not in any repository." -* `☿` means "I'm in a Mercurial repository." -* `±` means "I'm in a git repository." - -Those particular characters are meant to reflect the logos of the various -projects. You can of course change them to anything that makes sense to you. - -[mercurial]: {{links.mercurial}} -[git]: {{links.git}} - -To add this feature to your prompt you'll need to do two things in your theme -file. First, add the following function: - - :::bash - function prompt_char { - git branch >/dev/null 2>/dev/null && echo '±' && return - hg root >/dev/null 2>/dev/null && echo '☿' && return - echo '○' - } - -Then add `$(prompt_char)` somewhere inside your `PROMPT` variable -- wherever -you want the character to be displayed (probably at the end). - -Mercurial Repository Information --------------------------------- - -I use Mercurial at work and for my own personal projects and I find it very -handy to have some information about the current repository right in my -prompt. It keeps me "oriented," especially when I'm using [MQ][] and -manipulating the state of a repo. - -[MQ]: http://mercurial.selenic.com/wiki/MqExtension - -In the past I toyed around with some shell scripts to do this but eventually I -got fed up with the poor performance and horrible readability of that -solution. - -To fix this I wrote a small Mercurial extension called [hg-prompt][]. It adds -an `hg prompt` command to Mercurial that will let you output information about -the current repository. It's designed especially for use in shell prompts -(hence the name). - -I open sourced the code a while ago and since then a few people have made -feature requests and/or contributed patches. It's grown into a nice, flexible -little tool. You can check out the [documentation][hgpdoc] to see all the -things it can do. - -[hg-prompt]: /projects/hg-prompt/ -[hgpdoc]: http://sjl.bitbucket.org/hg-prompt/ - -I only use a few of the features myself: - -* The current repository branch. -* The "dirty" state of the repo (whether there are untracked/uncommitted files). -* Any tags pointing at the current revision. -* An "update" character to show whether running `hg update` would do anything. -* A list of MQ patches (if any are present). - -To put this into my prompt I've added a function to my theme: - - :::bash - function hg_prompt_info { - hg prompt --angle-brackets "\ - < on >\ - < at >\ - < - patches: >" 2>/dev/null - } - -Inside my `PROMPT` variable I use `$(hg_prompt_info)` to show the information. -The `2>/dev/null` makes sure no output is shown if I don't happen to be in a -Mercurial repository. - -Of course this isn't nearly as pretty or readable without color. Unfortunately -adding color codes makes the actual code almost unreadable: - - :::bash - function hg_prompt_info { - hg prompt --angle-brackets "\ - < on %{$fg[magenta]%}%{$reset_color%}>\ - < at %{$fg[yellow]%}%{$reset_color%}>\ - %{$fg[green]%}%{$reset_color%}< - patches: >" 2>/dev/null - } - -**Small note:** My favorite part of this prompt is that applied MQ patches are -orange while unapplied patches are grey. It makes it very obvious what's going -on. - -Showing all the MQ patches isn't something that will work for everyone. If you -work on repositories that ever have more than 5 or 6 patches at a time the -output is going to be too overwhelming. There are some other MQ-related -hg-prompt keywords that you might find useful though. - -Git Repository Information --------------------------- - -Unfortunately I can't *always* use Mercurial. Many projects that I contribute -to use git. - -I could try to use the [hg-git][] extension to let me use Mercurial to work -with them, but the extension just isn't fast or mature enough for my taste. -I've ended up buckling down and jumping down the rabbit hole that is git's -CLI. - -[hg-git]: http://hg-git.github.com/ - -Git doesn't seem to have any equivalent to hg-prompt, but I still wanted to -have some information in my prompt so I don't get disoriented. oh-my-zsh has a -bit of support for this, but it wasn't quite enough for me so I [hacked -something together][omzgitcommit] to add the last bit of functionality I -wanted. - -[omzgitcommit]: http://github.com/sjl/oh-my-zsh/commit/3d22ee248c6bce357c018a93d31f8d292d2cb4cd - -When I'm in a git repository my prompt now shows: - -* The current git branch. -* Whether there are any changes in the index. -* Whether there are any changes *not* in the index. - -Adding this to your prompt is very simple because oh-my-zsh has built-in -support for it. All you need to do is define a few variables in your theme: - - :::bash - ZSH_THEME_GIT_PROMPT_PREFIX=" on %{$fg[magenta]%}" - ZSH_THEME_GIT_PROMPT_SUFFIX="%{$reset_color%}" - ZSH_THEME_GIT_PROMPT_DIRTY="%{$fg[green]%}!" - ZSH_THEME_GIT_PROMPT_UNTRACKED="%{$fg[green]%}?" - ZSH_THEME_GIT_PROMPT_CLEAN="" - -Once you do that you can put `$(git_prompt_info)` into your `PROMPT` variable -to make it show up. - -**Note:** The `ZSH_THEME_GIT_PROMPT_UNTRACKED` variable is the part I added. -If your want to use that one (to be able to see indexed and unindexed changes -separately) you'll need to use [my fork of oh-my-zsh][omzfork]. - -[omzfork]: http://github.com/sjl/oh-my-zsh/ - -The Whole Thing ---------------- - -Here's what my new "prose" theme file currently looks like in its entirety: - - :::bash - function collapse_pwd { - echo $(pwd | sed -e "s,^$HOME,~,") - } - - function prompt_char { - git branch >/dev/null 2>/dev/null && echo '±' && return - hg root >/dev/null 2>/dev/null && echo '☿' && return - echo '○' - } - - function battery_charge { - echo `$BAT_CHARGE` 2>/dev/null - } - - function virtualenv_info { - [ $VIRTUAL_ENV ] && echo '('`basename $VIRTUAL_ENV`') ' - } - - function hg_prompt_info { - hg prompt --angle-brackets "\ - < on %{$fg[magenta]%}%{$reset_color%}>\ - < at %{$fg[yellow]%}%{$reset_color%}>\ - %{$fg[green]%}%{$reset_color%}< - patches: >" 2>/dev/null - } - - PROMPT=' - %{$fg[magenta]%}%n%{$reset_color%} at %{$fg[yellow]%}%m%{$reset_color%} in %{$fg_bold[green]%}$(collapse_pwd)%{$reset_color%}$(hg_prompt_info)$(git_prompt_info) - $(virtualenv_info)$(prompt_char) ' - - RPROMPT='$(battery_charge)' - - ZSH_THEME_GIT_PROMPT_PREFIX=" on %{$fg[magenta]%}" - ZSH_THEME_GIT_PROMPT_SUFFIX="%{$reset_color%}" - ZSH_THEME_GIT_PROMPT_DIRTY="%{$fg[green]%}!" - ZSH_THEME_GIT_PROMPT_UNTRACKED="%{$fg[green]%}?" - ZSH_THEME_GIT_PROMPT_CLEAN="" - -The file is also [on GitHub][prosetheme]. Remember, you'll need -[my fork of oh-my-zsh][omzfork] to use the `ZSH_THEME_GIT_PROMPT_UNTRACKED` -feature. - -[prosetheme]: http://github.com/sjl/oh-my-zsh/blob/master/themes/prose.zsh-theme - -A shell prompt is something that we'll each see thousands of times a day, so -it's something that you should take the time to customize into a useful tool. -Feel free to use my prompt as a starting point or just as a source of ideas. - -If you have any questions, suggestions, or examples of your own please fine me on -[Twitter][twsl] -- I'd love to hear them! - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2010/02/my-extravagant-zsh-prompt.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2010/02/my-extravagant-zsh-prompt.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,494 @@ ++++ +title = "My Extravagant Zsh Prompt" +snip = "It’s big, but my monitor isn’t running out of ink." +date = 2010-02-01T01:05:00Z +draft = false + ++++ + +I spend a lot of time in a Terminal window at a command line. Up until about a +month ago I was using [bash][] for my shell. I decided to try switching to +[Zsh][] after hearing a lot of good things about it and I'm very happy with +the change. + +A few days ago I [tweeted][] my current Zsh prompt and the general response +was: "Cool, but how did you do it?" I promised to write more about it when I +got some free time, and it looks like that time is now. + +**One quick note:** This entry is about the prompt that *I* find useful. You +are not me, so you'll almost certainly have different needs. That's great! +Take this prompt and hack it to make whatever works for you. If you're feeling +generous you should find me on [Twitter][twsl] and show me what you changed -- +I might want to steal/appropriate your changes. + +**Another quick note:** I've customized the colors of my Terminal. They're +based on the [Monokai][] TextMate theme, and I think they look very nice. The +colors will be different for you. If you want colors like mine you should take +a look at the [entry][candy] I wrote about it. + +[bash]: http://en.wikipedia.org/wiki/Bash +[Zsh]: http://www.zsh.org/ +[tweeted]: http://twitter.com/stevelosh/status/8259755151 +[Monokai]: http://www.monokai.nl/blog/2006/07/15/textmate-color-theme/ +[candy]: /blog/2009/03/candy-colored-terminal/ +[twsl]: {{links.twsl}} + +{{% toc %}} + +Why Should You Care? +-------------------- + +Many people use the command line every day and never bother to customize their +prompts. It's just a bit of text that's printed before every command — why +should you waste time learning how to customize it? + +I feel that the *most* important aspect of my command line work is the prompt. +Your prompt is something you'll see *literally* thousands of times a day. Why +not take 30 minutes and customize it into something that's much more useful? + +I firmly believe I'm right in thinking this way. 30 minutes (or even 3 hours) +of customization to make your work easier for the **rest of your life** (or at +least however long you stay with your current shell) *is* worth it. + +This Entry is About Zsh Prompts +------------------------------- + +As I mentioned earlier I now use Zsh as my command line shell. If you use bash +(the default on most modern systems) the syntax to create the shell will be +different. + +Really, though, you should give Zsh a try. + +Screenshots +----------- + +I can write all day, but in the end I think a screenshot will be more helpful +than anything I write. + +Here's a sample of my current Zsh prompt: + +![My Zsh Prompt](/media/images/blog/2009/02/zsh-prompt.png "My Zsh Prompt") + +And here's a version of that screenshot with some comments added to explain +things: + +![My Zsh Prompt with Comments](/media/images/blog/2009/02/zsh-prompt-comments.png "My Zsh Prompt with Comments") + +If you want to know *how* I created that prompt, read on! + +Oh My Zsh! +---------- + +The first thing I'd do when starting out with Zsh is to install +[robbyrussell's][robr] [oh-my-zsh][omz]. It's a great collection of very +useful Zsh configurations and aliases which set some sane defaults and make +working with Zsh much nicer. + +The instructions for installing it are on the project page. + +[robr]: http://github.com/robbyrussell +[omz]: http://github.com/robbyrussell/oh-my-zsh + +Define a "Theme" +---------------- + +oh-my-zsh uses [theme files][themes] to specify how your prompt looks. + +An oh-my-zsh theme file is just a Zsh script which sets a few variables that +the oh-my-zsh scripts use to render your prompt. + +Go ahead and create a new theme file for your prompt. Call it whatever you +like. I called my theme "prose" because it's more verbose than the others. +Copy the contents of one of the other themes to get started. + +The `PROMPT` variable is what you'll be modifying to change how your prompt +looks. That's the `PROMPT=whatever` line of the theme file. + +Once you're done making your awesome prompt you should fork oh-my-zsh on +GitHub and commit/push your new theme so other people can see and use your +prompt. + +[themes]: http://github.com/robbyrussell/oh-my-zsh/tree/master/themes/ + +Username and Hostname +--------------------- + +The first two pieces of my prompt are the simplest: username and hostname. I +SSH between machines pretty frequently so I find it nice to have these in my +prompt to remind me of where I am. + +These are things that you'll find in many, many Zsh prompts. For more +information about this kind of stuff check out [this page][zshpr]. + +After adding in the colors this piece of my `PROMPT` variable looks like this: + +```text +%{$fg[magenta]%}%n%{$reset_color%} at %{$fg[yellow]%}%m%{$reset_color%} +``` + +[zshpr]: http://www.acm.uiuc.edu/workshops/zsh/prompt/escapes.html + +Current Directory +----------------- + +I like to have the current working directory displayed in my prompt. Zsh has +two built-in ways to show the current directory: + +* **%d** shows the *entire* path. +* **%~** shows the path with any variables replaced. + +Unfortunately neither of these work for me. + +If I'm in a directory that's under my home directory I want to see `~` instead +of the full path. This would seem to mean that I should use `%~`, but there's +a catch. + +The `%~` sequence also replaces any directories that happen to be environment +variables. That means if I'm in something under my `virtualenvs` directory +I'll see `~WORKON_HOME/venv_name` which is extremely ugly. + +To get around this I defined a simple function that implements the behavior I +want. + +To add this to your prompt you'll first need to add the following function to +your theme: + +```bash +function collapse_pwd { + echo $(pwd | sed -e "s,^$HOME,~,") +} +``` + +Then add `$(collapse_pwd)` to your `PROMPT` variable wherever you'd like the +directory to show up. + +After adding in the colors this piece of my `PROMPT` variable looks like this: + +```text +in %{$fg_bold[green]%}$(collapse_pwd)%{$reset_color%} +``` + +**UPDATE:** In a comment Juliano pointed out a better way to do this. I've +gotten rid of the `collapse_pwd` function and put this into my `PROMPT` +variable instead: + +```bash +${PWD/#$HOME/~} +``` + +My Right-Prompt: Battery Capacity +--------------------------------- + +The next piece of my prompt I'll talk about is the most stand-alone one: the +battery charge. + +I work exclusively on laptops. I don't own a desktop machine — I have a +Macbook, Macbook Pro and Asus EEE PC and my work machine is a Macbook Pro. I +work from coffeeshops and client meetings pretty often, so it's nice to have a +reminder of my remaining battery power to know when I need to plug in. + +On the right-hand side of the prompt in the screenshot you can see a series of +green triangles. These represent my current battery charge and they turn to +empty triangles as the battery becomes depleted. Once the battery reaches 60% +they turn yellow and once it reaches 40% they turn red. + +To do this I wrote a *very* simple Python script to output my current battery +capacity. The entire script is shown below. + +```python +#!/usr/bin/env python +# coding=UTF-8 + +import math, subprocess + +p = subprocess.Popen(["ioreg", "-rc", "AppleSmartBattery"], stdout=subprocess.PIPE) +output = p.communicate()[0] + +o_max = [l for l in output.splitlines() if 'MaxCapacity' in l][0] +o_cur = [l for l in output.splitlines() if 'CurrentCapacity' in l][0] + +b_max = float(o_max.rpartition('=')[-1].strip()) +b_cur = float(o_cur.rpartition('=')[-1].strip()) + +charge = b_cur / b_max +charge_threshold = int(math.ceil(10 * charge)) + +# Output + +total_slots, slots = 10, [] +filled = int(math.ceil(charge_threshold * (total_slots / 10.0))) * u'▸' +empty = (total_slots - len(filled)) * u'▹' + +out = (filled + empty).encode('utf-8') +import sys + +color_green = '%{%}' +color_yellow = '%{%}' +color_red = '%{%}' +color_reset = '%{%}' +color_out = ( + color_green if len(filled) > 6 + else color_yellow if len(filled) > 4 + else color_red +) + +out = color_out + out + color_reset +sys.stdout.write(out) +``` + +**Note:** The `color_*` variables contain UTF-8 characters which don't show up +in a normal web browser. They *should* be preserved when copying and pasting +from a decent browser. If the colors don't work find me on [Twitter][twsl] and +I'll put the raw script up somewhere so you can download it. + +**UPDATE:** [ehamberg][] [sent me][ehbtweet] the beginnings of a +Linux-compatible battery capacity script. If you flesh it out and make it into +a complete script please let me know and I'll add it here. + +[ehamberg]: http://twitter.com/ehamberg/ +[ehbtweet]: http://twitter.com/ehamberg/status/8503213690 + +I've put this script in a file named `batcharge.py` in my `~/bin/` directory. +You can of course name it and place it anything/anywhere you like. + +No, it's not perfect. No, I don't care. It gets the job done and any +brainpower I could spend on making it more efficient could be better spent on +writing or working on something else. + +If you want to improve it, feel free and *please* send me the link to your work on +[Twitter][twsl] so I can use it! + +Now that you've got the script, it's time to add it to your prompt. + +If you want this to appear on the right-hand side of your screen (like mine +does) you'll need to add two things to your theme file. The first is a +function that calls the script: + +```bash +function battery_charge { + echo `$BAT_CHARGE` 2>/dev/null +} +``` + +After that you'll need to define the Zsh `RPROMPT` variable: + +```bash +RPROMPT='$(battery_charge)' +``` + +Once those are in your theme file any new Terminal windows you open should +show the battery capacity to the right of your prompt! + +Repository Types +---------------- + +In my personal work and my full-time job I work with both +[Mercurial][mercurial] and [git][] repositories. I find it helpful to have a +visual reminder of what kind of repository I'm working in. + +To do this I've replaced the standard `$` character in my prompt with +different Unicode characters depending on what kind of repository I'm +currently in: + +* `○` means "I'm not in any repository." +* `☿` means "I'm in a Mercurial repository." +* `±` means "I'm in a git repository." + +Those particular characters are meant to reflect the logos of the various +projects. You can of course change them to anything that makes sense to you. + +[mercurial]: {{links.mercurial}} +[git]: {{links.git}} + +To add this feature to your prompt you'll need to do two things in your theme +file. First, add the following function: + +```bash +function prompt_char { + git branch >/dev/null 2>/dev/null && echo '±' && return + hg root >/dev/null 2>/dev/null && echo '☿' && return + echo '○' +} +``` + +Then add `$(prompt_char)` somewhere inside your `PROMPT` variable — wherever +you want the character to be displayed (probably at the end). + +Mercurial Repository Information +-------------------------------- + +I use Mercurial at work and for my own personal projects and I find it very +handy to have some information about the current repository right in my +prompt. It keeps me "oriented," especially when I'm using [MQ][] and +manipulating the state of a repo. + +[MQ]: http://mercurial.selenic.com/wiki/MqExtension + +In the past I toyed around with some shell scripts to do this but eventually I +got fed up with the poor performance and horrible readability of that +solution. + +To fix this I wrote a small Mercurial extension called [hg-prompt][]. It adds +an `hg prompt` command to Mercurial that will let you output information about +the current repository. It's designed especially for use in shell prompts +(hence the name). + +I open sourced the code a while ago and since then a few people have made +feature requests and/or contributed patches. It's grown into a nice, flexible +little tool. You can check out the [documentation][hgpdoc] to see all the +things it can do. + +[hg-prompt]: /projects/hg-prompt/ +[hgpdoc]: http://sjl.bitbucket.org/hg-prompt/ + +I only use a few of the features myself: + +* The current repository branch. +* The "dirty" state of the repo (whether there are untracked/uncommitted files). +* Any tags pointing at the current revision. +* An "update" character to show whether running `hg update` would do anything. +* A list of MQ patches (if any are present). + +To put this into my prompt I've added a function to my theme: + +```bash +function hg_prompt_info { + hg prompt --angle-brackets "\ +< on >\ +< at >\ +< +patches: >" 2>/dev/null +} +``` + +Inside my `PROMPT` variable I use `$(hg_prompt_info)` to show the information. +The `2>/dev/null` makes sure no output is shown if I don't happen to be in a +Mercurial repository. + +Of course this isn't nearly as pretty or readable without color. Unfortunately +adding color codes makes the actual code almost unreadable: + +```bash +function hg_prompt_info { + hg prompt --angle-brackets "\ +< on %{$fg[magenta]%}%{$reset_color%}>\ +< at %{$fg[yellow]%}%{$reset_color%}>\ +%{$fg[green]%}%{$reset_color%}< +patches: >" 2>/dev/null +} +``` + +**Small note:** My favorite part of this prompt is that applied MQ patches are +orange while unapplied patches are grey. It makes it very obvious what's going +on. + +Showing all the MQ patches isn't something that will work for everyone. If you +work on repositories that ever have more than 5 or 6 patches at a time the +output is going to be too overwhelming. There are some other MQ-related +hg-prompt keywords that you might find useful though. + +Git Repository Information +-------------------------- + +Unfortunately I can't *always* use Mercurial. Many projects that I contribute +to use git. + +I could try to use the [hg-git][] extension to let me use Mercurial to work +with them, but the extension just isn't fast or mature enough for my taste. +I've ended up buckling down and jumping down the rabbit hole that is git's +CLI. + +[hg-git]: http://hg-git.github.com/ + +Git doesn't seem to have any equivalent to hg-prompt, but I still wanted to +have some information in my prompt so I don't get disoriented. oh-my-zsh has a +bit of support for this, but it wasn't quite enough for me so I [hacked +something together][omzgitcommit] to add the last bit of functionality I +wanted. + +[omzgitcommit]: http://github.com/sjl/oh-my-zsh/commit/3d22ee248c6bce357c018a93d31f8d292d2cb4cd + +When I'm in a git repository my prompt now shows: + +* The current git branch. +* Whether there are any changes in the index. +* Whether there are any changes *not* in the index. + +Adding this to your prompt is very simple because oh-my-zsh has built-in +support for it. All you need to do is define a few variables in your theme: + +```bash +ZSH_THEME_GIT_PROMPT_PREFIX=" on %{$fg[magenta]%}" +ZSH_THEME_GIT_PROMPT_SUFFIX="%{$reset_color%}" +ZSH_THEME_GIT_PROMPT_DIRTY="%{$fg[green]%}!" +ZSH_THEME_GIT_PROMPT_UNTRACKED="%{$fg[green]%}?" +ZSH_THEME_GIT_PROMPT_CLEAN="" +``` + +Once you do that you can put `$(git_prompt_info)` into your `PROMPT` variable +to make it show up. + +**Note:** The `ZSH_THEME_GIT_PROMPT_UNTRACKED` variable is the part I added. +If your want to use that one (to be able to see indexed and unindexed changes +separately) you'll need to use [my fork of oh-my-zsh][omzfork]. + +[omzfork]: http://github.com/sjl/oh-my-zsh/ + +The Whole Thing +--------------- + +Here's what my new "prose" theme file currently looks like in its entirety: + +```bash +function collapse_pwd { + echo $(pwd | sed -e "s,^$HOME,~,") +} + +function prompt_char { + git branch >/dev/null 2>/dev/null && echo '±' && return + hg root >/dev/null 2>/dev/null && echo '☿' && return + echo '○' +} + +function battery_charge { + echo `$BAT_CHARGE` 2>/dev/null +} + +function virtualenv_info { + [ $VIRTUAL_ENV ] && echo '('`basename $VIRTUAL_ENV`') ' +} + +function hg_prompt_info { + hg prompt --angle-brackets "\ +< on %{$fg[magenta]%}%{$reset_color%}>\ +< at %{$fg[yellow]%}%{$reset_color%}>\ +%{$fg[green]%}%{$reset_color%}< +patches: >" 2>/dev/null +} + +PROMPT=' +%{$fg[magenta]%}%n%{$reset_color%} at %{$fg[yellow]%}%m%{$reset_color%} in %{$fg_bold[green]%}$(collapse_pwd)%{$reset_color%}$(hg_prompt_info)$(git_prompt_info) +$(virtualenv_info)$(prompt_char) ' + +RPROMPT='$(battery_charge)' + +ZSH_THEME_GIT_PROMPT_PREFIX=" on %{$fg[magenta]%}" +ZSH_THEME_GIT_PROMPT_SUFFIX="%{$reset_color%}" +ZSH_THEME_GIT_PROMPT_DIRTY="%{$fg[green]%}!" +ZSH_THEME_GIT_PROMPT_UNTRACKED="%{$fg[green]%}?" +ZSH_THEME_GIT_PROMPT_CLEAN="" +``` + +The file is also [on GitHub][prosetheme]. Remember, you'll need +[my fork of oh-my-zsh][omzfork] to use the `ZSH_THEME_GIT_PROMPT_UNTRACKED` +feature. + +[prosetheme]: http://github.com/sjl/oh-my-zsh/blob/master/themes/prose.zsh-theme + +A shell prompt is something that we'll each see thousands of times a day, so +it's something that you should take the time to customize into a useful tool. +Feel free to use my prompt as a starting point or just as a source of ideas. + +If you have any questions, suggestions, or examples of your own please fine me on +[Twitter][twsl] — I'd love to hear them! + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2010/04/a-faster-feed-apart.html --- a/content/blog/2010/04/a-faster-feed-apart.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,449 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "A Faster Feed Apart" - snip: "Rethinking A Feed Apart’s backend." - created: 2010-04-30 22:55:00 -%} - -{% block article %} - -[An Event Apart][aea] is a conference for web developers and designers that -happens a few times a year in various cities. [A Feed Apart][afa] is a site -that aggregates tweets during each conference and displays them in a live -stream so attendees can follow them during the conference, and people not -attending can see what the attendees are talking about. - -A Feed Apart was originally written by [Nick Sergeant][nick] and [Pete -Karl][pete] of [Lion Burger][lionburger] during one of the conferences. Since -then a lot of people have used it and love it. - -The current A Feed Apart site is not without its problems. It was written in -a single night so it's not perfect. During the last conference it went down -several times and lost some tweets along the way. - -I work at [Dumbwaiter Design][dwaiter] with Nick and one day he mentioned that -it would be cool if we rewrote A Feed Apart from the ground up. He's learned -a lot about how people use the site and what the big problems are, so we'd have -a better idea of what we need to accomplish. - -Nick, myself and [Ali Ali][ali] have risen to the challenge and started -rewriting A Feed Apart. Ali is a designer and is taking care of the design of -the new site. Nick is handling all the frontend HTML, CSS and Javascript. My -job is the backend. - -Ali posted a [blog entry][aliblog] about the design of the new version so -I figured I'd write about the backend to show how I'm trying to improve it. If -you have any advice I'd love to hear it -- the next An Event Apart conference -is about three weeks away so there's still time to add improvements. - -[aea]: http://aneventapart.com/ -[afa]: http://afeedapart.com/ -[nick]: http://nicksergeant.com/ -[pete]: http://pkarl.com/ -[lionburger]: http://lionburger.com/ -[dwaiter]: http://dwaiter.com/ -[ali]: http://alialithinks.com/ -[aliblog]: http://www.dcamm.com/blog/archives/765 - -[TOC] - -How AFA is Used ---------------- - -You can't hope to improve on a site unless you know how people are going to use -it. AFA has been around for a while and Nick has learned a lot about what -people want to get out of it. - -There are three main kinds of users of AFA: - -* People attending the conference -* People that wish they were attending the conference -* The speakers/organizers of the conference - -### People Attending the Conference - -People that attend AEA are web developers and designers. They're up-to-date on -the latest technology and almost all of them use [Twitter][]. - -During the conference you'll see a sea of laptops in the audience. Attendees -will tweet about whatever is happening *right now*. There's a huge amount of -conversation that happens between attendees that AFA is trying to collect and -present. - -A simple example is one that Nick mentioned to me: if a presenter mentions -a website during her presentation someone will tweet the URL so other people -can have a link to easily click on. Attendees will also tweet their agreement -or disagreement with what the presenter is saying *as they're speaking*. - -The most important aspect of AFA for this group of people is the *real time -conversation*. If AFA doesn't show tweets until a minute after they've been -posted it's useless to these people. - -Another important part of AFA for attendees is that it's a one-stop-shop for -the conversation behind AEA. Switching between windows/tabs/applications to -read and contribute is annoying. - -### People That Wish They Were Attending the Conference - -The next group of users is those that can't attend the conference for a variety -of reasons but are still interested in what's going on. - -For this group the real time nature of AFA is unimportant. They're probably -not going to be following the conversation 24/7 so if a tweet takes a few -minutes to display it's not a big deal. - -What these people *do* care about is the integrity of the stream. They don't -want to miss any part of the conversation. If AFA loses tweets they're better -off doing a simple search on Twitter because they'll get the whole story. - -### Speakers and Organizers - -The last main group of users is the speakers and organizers of AEA. - -Any speaker worth their salt will want to know what people are said about their -presentation. Obviously they can't be reading AFA while they're presenting, -but afterword they'll certainly want to go back and see what people were saying -during their specific presentation. - -Likewise, organizers want to know which presentations people liked and which -ones people didn't enjoy. This could easily influence who they choose to -invite to future conferences. - -For this group the most important aspect of AFA is the organization of the -conversation into chunks, each of which applies to a single presentation. By -reading through each chunk of conversation they can get an idea of the general -response to each presentation. - -[Twitter]: http://twitter.com/ - -Goals ------ - -Once we identified the main users of AFA we were able to come up with the goals -for the new version of the site: - -* **Stay sane while developing.** Ali, Nick and myself are rewriting the site - as a side project, so we don't want it to take *too* much of our time or cause - *too* much stress. - -* **Provide the real time conversation.** The site needs to be fast and - responsive, so people at the conferences can use it to converse. - -* **Don't miss anything.** People following along from home shouldn't feel - like they're missing anything. We want to provide a *complete* version of the - conversation happening behind the event. - -* **Organize the conversation.** Presenters and speakers need a way to see - what people are saying about them. They want to know what people think about - each "chunk" of the event. - -* **Grease the wheels of (physical) social interaction.** This is something - that AFA hasn't tried to address before, but that we'd like to work on with - this version. There are a lot of people at each conference and we'd like to - help them get together. Whether it's going out for dinner or meeting up at the - [Media Temple][mt] party we want to get people talking to each other. I won't - talk about this goal in this post because we're still figuring out the best way - to do it. - -[mt]: http://mediatemple.net/ - -Staying Sane ------------- - -If the three of us tried to create the site with nothing more than a couple of -laptops and a few chats in person we'd go crazy. We use a few tools to help us -manage the development of the site. - -To create **wireframes of the design** we're using a free account at [Hot -Gloo][hotgloo]. Hot Gloo is a great tool that lets us quickly sketch out ideas -and comment on them. - -To share **design comps** we're using [Dropbox][]. It's simple to set up -a shared Dropbox folder and Nick and I can get real time updates when Ali makes -changes to the design. - -To **work together on the code** Nick and I use [Mercurial][] repositories. -Mercurial lets us work on the same code bases simultaneously and we almost -never have to worry about merging. We use [Codebase][] for hosting and issue -tracking. - -[hotgloo]: http://hotgloo.com/ -[Dropbox]: http://dropbox.com/ -[Mercurial]: http://hg-scm.org/ -[Codebase]: http://codebasehq.com/ - -Being Real Time ---------------- - -The previous version of AFA wasn't *truly* real time. When you went to the -site your browser would ask AFA for the newest updates every 10 seconds. - -There are two main problems with this approach. The first is that it's not -really *real time*. I've noticed this being an issue in my own experience. - -When I watch any of the various "live streams" of [Apple][] press conferences -I'm usually at work where other people are also watching. We very rarely load -the pages of the streaming sites like Gizmodo at the exact same second, so our -browsers will be out of sync with each other. Nick might get an update that -I would have to wait 8 more seconds to see. - -When I glance over at his screen and see an update that I don't have -I instinctively refresh the page to get it. This defeats the purpose of the -"live updating" code that the developers of these sites worked on. They may as -well have just made a static page and told me to refresh. - -The second problem is that querying every 10 seconds can be taxing on the -site's database. We're doing this as a side project so we don't have unlimited -funding for a hefty database server. If 1,000 people are querying for updates -every 10 seconds that's 100 requests per second to the database. This means we -need to have some kind of in-memory cacheing if we want the site to feel snappy -on modest hardware. - -We want the new AFA to be truly real time. To do this we need to use [long -polling][longpolling] by users' browsers to wait for updates and return them as -soon as they come in. We also need to retrieve the updates as fast as we -possibly can, and they need to be stored in memory to avoid hitting the -database constantly. - -[Apple]: http://apple.com/ -[longpolling]: http://en.wikipedia.org/wiki/Push_technology#Long_polling - -### Retrieving Updates - -The bulk of the conversation about AEA conferences comes from Twitter. Tweets -are the most important items that we need to display on the site, so we're -using Twitter's streaming API to pull them in. - -Since we don't want to tie up an entire server to pull in tweets I've decided -to create a [Diesel][]-based application called **The Nozzletron** to parse the -streaming API. Diesel is a [Python][] framework that takes an elegant approach -to asynchronous communication with clients and servers. - -Twitter's streaming API accepts HTTP requests and returns "chunked" responses, -each of which is a tweet. Unfortunately the Diesel's built-in HTTP client -doesn't handle chunked HTTP responses so I had to write some code to handle -them myself. - -The Nozzletron will connect to Twitter's streaming API and wait for data to -come in. If there's no data to process it will relinquish the server's -processor so it can do other things. Processing a single tweet that comes in -doesn't take much time, so the server is free to do other things most of the -time. - -I've also created another application called **The Flickrtron** to pull in -[Flickr][] photos. Unfortunately Flickr doesn't have a streaming API like -Twitter so I have to resort to polling Flickr's API every few minutes for new -photos. Flickr is much less of a real time medium than Twitter though, so -I don't think this is a very big problem. - -I'm using a Python library called [Beej's Flickr API][flickrapi] to talk to -Flickr. It is horrible. It calls itself an "API" but is really just a thin -wrapper around calls to the Flickr API. The objects it returns for API calls -are Elementtree objects representing the XML of the response. - -I wish I could do something like: - - :::python - thumb_url = photo.thumbnail_url - -Instead I have to use this monstrosity: - - :::python - thumb_url = "http://farm%s.static.flickr.com/%s/%s_%s_s.jpg" % ( - photo.get('farm'), photo.get('server'), photo.get('id'), - photo.get('secret'), - ) - -I wish there were a better Python Flickr API out there but there doesn't seem -to be one. If I've missed it please let me know! - -[Diesel]: http://dieselweb.org/ -[Python]: http://python.org/ -[Flickr]: http://flickr.com/ -[flickrapi]: http://stuvel.eu/projects/flickrapi - -### Storing Updates - -We need a fast way to store and retrieve updates so AFA can provide a real time -view of the conversation happening at AEA. With this in mind I've chosen -[Redis][] to store the updates of the currently-happening event. - -Redis is an easy-to-set-up, disgustingly-fast, in-memory data store. It's -similar to [memcached][] but has more intelligent data structures that make my -life easier for this project. - -As updates (tweets and flickr photos) are scraped by The Nozzletron and The -Flickrtron they're placed at the tail end of a Redis list. That means I can -quickly and easily get all the items since item `N` when a user's browser -requests them with a single `LRANGE {list-key} N -1` call. - -I'm also keeping a few other pieces of information in Redis. For example, -there's a set of photo IDs held in the `{item-key}:flickrtron:grabbed_photos` -key that keeps track of all of the photos we've already seen. - -This makes it easy to tell if we've already seen a photo (and therefore don't -need to query Flickr for more information) -- it's a simple `SISMEMBER -{item-key}:flickrtron:grabbed_photos {photo-id}` call. - -I'm also using Redis to store statistics about the site like: - -* How many tweets we've scraped. -* How many photos we've scraped. -* How many people are currently waiting for updates. - -This kind of information will be extremely valuable in the future when we're -planning improvements to the site. Redis makes it fast and safe to update this -information using the `INCRBY` and `DECR` commands. - -There's one more component to The Nozzletron and The Flickrton that I haven't -mentioned. Both use Redis' `PUBLISH` command to push new updates out to users' -browsers as they arrive. I'll talk more about that in the next section. - -[Redis]: http://code.google.com/p/redis/ -[memcached]: http://memcached.org/ - -### Sending Updates to Users - -As I mentioned before we want to send updates to users *as soon as they're -received*. To do this I've created another Diesel-based application called -**Halley** to handle this [Comet][]-style communication. - -Halley has a few components. The first uses Diesel's [Redis API][dieselredis] -to subscribe to a Redis channel like `live:{event-id}:items` and -[fire][dieselfire] off messages whenever something new comes in. As soon as -a new update comes in from The Nozzletron or The Flickrtron all of Halley's -clients will get it. - -When I started working on AFA Diesel's Redis client didn't support the very new -`PUBLISH`/`SUBSCRIBE`/`UNSUBSCRIBE` commands. I implemented them, put my -changes on [BitBucket][], sent a pull request, and started talking to the -Diesel crew on IRC. - -One of the maintainers pulled my patches and made them even better. Now -Diesel's Redis client has full `PUBLISH`/`SUBSCRIBE`/`UNSUBSCRIBE` support. -It's a great example of how open source projects can produce awesome results. - -The other main component of Halley is an HTTP server that listens for -connections from browsers. The Javascript on the site will call Halley and say -"I need updates since number `X`", where `X` is the length of the Redis list of -updates at the last time it spoke to the server. - -Halley uses Diesel's [HTTP Server][dieselhttp] to manage these requests. - -If a client is asking for everything since `X` and `X` is a smaller number than -the current number of items it will return the updates that have happened since -then. This might happen if we return an update and then two more updates -happen before the browser gets around to sending another request. - -If a client is asking for everything since `X` and `X` is equal to the current -number of items Halley will wait for a new message to be fired from the -[Loop][dieselloop] that's watching the Redis channel. While Halley waits she -relinquishes the processor to the server so other requests can be handled. - -There's a bit of code to prevent DoS attacks that request every item in the -queue over and over again, of course. - -[Comet]: http://en.wikipedia.org/wiki/Comet_(programming) -[dieselredis]: http://bitbucket.org/boomplex/diesel/src/tip/diesel/protocols/redis.py -[dieselfire]: http://bitbucket.org/boomplex/diesel/src/tip/diesel/core.py#cl-90 -[BitBucket]: http://bitbucket.org/ -[dieselhttp]: http://bitbucket.org/boomplex/diesel/src/tip/diesel/protocols/http.py#cl-156 -[dieselloop]: http://bitbucket.org/boomplex/diesel/src/tip/diesel/core.py#cl-179 - -Organizing the Conversation ---------------------------- - -The live stream is an important component of AFA, but it's not the only one. -We also need to organize updates into logical chunks by event and presentation, -and provide archives of old events so people can see what happened. - -The main AFA site is built with [Django][] and served with [Gunicorn][] and -[Nginx][]. It uses a [Postgresql][] database to store data that's not "live". -Because queries for live data are handled with Diesel and Redis we don't need -to send those request through the full Django/Postgresql stack. Django and -Postgresql are only involved when you load a fresh page, and they're *more* -than capable of handling the amount of traffic that AFA gets for those kind of -requests. - -I've created an application called **The Strainer** to copy data from the live -stream to the Postgresql database. The Strainer looks at the list of live -items in Redis, parses those items into Django models and saves them to the -Postgresql database. - -Using two different types of stores (Redis and Postgresql) means we can get the -best of both worlds for AFA: - -* We can keep the live data that's accessed *constantly* in an in-memory Redis - datastore which makes it blazingly fast. - -* We can keep the less-frequently-used data in an on-disk Postgresql database - which lets us to keep our memory usage low and hosting costs down. - -Django's models and managers make it very easy to separate updates out into the -various sessions and presentations that happen at the conferences. - -Sessions and presentations are much less in-demand than the live stream so we -can take advantage of Django's abstractions without worrying about the extra -memory/CPU usage we incur by doing so. - -[Django]: http://djangoproject.com/ -[Gunicorn]: http://gunicorn.org/ -[Nginx]: http://nginx.org/ -[Postgresql]: http://www.postgresql.org/ - -Staying Consistent ------------------- - -Another problem AFA has faced in the past is losing tweets. No code is perfect -(and mine *certainly* is not) so we need to anticipate that some of the -applications we're using will crash at some point. - -I'm using [Supervisord][] to monitor the various processes of AFA on the -server. If a process crashes for some reason it will be restarted -automatically. - -Supervisord also has a wonderful Python API, so I've created a simple Dashboard -view in the Django site that lets us stop/start/restart each individual process -with a simple web interface. The dashboard also shows us the current memory -usage of the server and some other statistics so we can monitor how things are -working through a web browser (instead of SSH'ing into the server, which is -a pain on a phone). - -[Supervisord]: http://supervisord.org/ - -Getting Bigger --------------- - -An Event Apart is a large event, but it's not a *huge* event. Despite this -I've been trying to build the backend in a way that can be easily scaled. - -Right now it's hosted on a single server, but each of the individual components -could be moved to a separate server with less than an hour of work each: - -* Redis -* Postgresql -* Django, Nginx and Gunicorn -* The Nozzletron -* The Flickrtron -* The Strainer -* Halley - -Moving Django/Gunicorn, Nginx, Redis, Halley, and Postgresql to dedicated -servers would increase the performance of the site *immensely*. I can't -imagine an event that would provide enough traffic to require more than that. - -Even if there were an event that needed that kind of throughput, we could -easily split the single Halley and Redis servers into multiple load-balanced -servers. - -A Work in Progress ------------------- - -This new version of A Feed Apart is still being built. I'm learning new things -every time I work on it, and I'm sure there's still room for improvement. - -If you have any questions, advice, or want me to go more in-depth about -a specific aspect of the site's backend please let me know! - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2010/04/a-faster-feed-apart.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2010/04/a-faster-feed-apart.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,448 @@ ++++ +title = "A Faster Feed Apart" +snip = "Rethinking A Feed Apart’s backend." +date = 2010-04-30T22:55:00Z +draft = false + ++++ + +[An Event Apart][aea] is a conference for web developers and designers that +happens a few times a year in various cities. [A Feed Apart][afa] is a site +that aggregates tweets during each conference and displays them in a live +stream so attendees can follow them during the conference, and people not +attending can see what the attendees are talking about. + +A Feed Apart was originally written by [Nick Sergeant][nick] and [Pete +Karl][pete] of [Lion Burger][lionburger] during one of the conferences. Since +then a lot of people have used it and love it. + +The current A Feed Apart site is not without its problems. It was written in +a single night so it's not perfect. During the last conference it went down +several times and lost some tweets along the way. + +I work at [Dumbwaiter Design][dwaiter] with Nick and one day he mentioned that +it would be cool if we rewrote A Feed Apart from the ground up. He's learned +a lot about how people use the site and what the big problems are, so we'd have +a better idea of what we need to accomplish. + +Nick, myself and [Ali Ali][ali] have risen to the challenge and started +rewriting A Feed Apart. Ali is a designer and is taking care of the design of +the new site. Nick is handling all the frontend HTML, CSS and Javascript. My +job is the backend. + +Ali posted a [blog entry][aliblog] about the design of the new version so +I figured I'd write about the backend to show how I'm trying to improve it. If +you have any advice I'd love to hear it — the next An Event Apart conference +is about three weeks away so there's still time to add improvements. + +[aea]: http://aneventapart.com/ +[afa]: http://afeedapart.com/ +[nick]: http://nicksergeant.com/ +[pete]: http://pkarl.com/ +[lionburger]: http://lionburger.com/ +[dwaiter]: http://dwaiter.com/ +[ali]: http://alialithinks.com/ +[aliblog]: http://www.dcamm.com/blog/archives/765 + +{{% toc %}} + +How AFA is Used +--------------- + +You can't hope to improve on a site unless you know how people are going to use +it. AFA has been around for a while and Nick has learned a lot about what +people want to get out of it. + +There are three main kinds of users of AFA: + +* People attending the conference +* People that wish they were attending the conference +* The speakers/organizers of the conference + +### People Attending the Conference + +People that attend AEA are web developers and designers. They're up-to-date on +the latest technology and almost all of them use [Twitter][]. + +During the conference you'll see a sea of laptops in the audience. Attendees +will tweet about whatever is happening *right now*. There's a huge amount of +conversation that happens between attendees that AFA is trying to collect and +present. + +A simple example is one that Nick mentioned to me: if a presenter mentions +a website during her presentation someone will tweet the URL so other people +can have a link to easily click on. Attendees will also tweet their agreement +or disagreement with what the presenter is saying *as they're speaking*. + +The most important aspect of AFA for this group of people is the *real time +conversation*. If AFA doesn't show tweets until a minute after they've been +posted it's useless to these people. + +Another important part of AFA for attendees is that it's a one-stop-shop for +the conversation behind AEA. Switching between windows/tabs/applications to +read and contribute is annoying. + +### People That Wish They Were Attending the Conference + +The next group of users is those that can't attend the conference for a variety +of reasons but are still interested in what's going on. + +For this group the real time nature of AFA is unimportant. They're probably +not going to be following the conversation 24/7 so if a tweet takes a few +minutes to display it's not a big deal. + +What these people *do* care about is the integrity of the stream. They don't +want to miss any part of the conversation. If AFA loses tweets they're better +off doing a simple search on Twitter because they'll get the whole story. + +### Speakers and Organizers + +The last main group of users is the speakers and organizers of AEA. + +Any speaker worth their salt will want to know what people are said about their +presentation. Obviously they can't be reading AFA while they're presenting, +but afterword they'll certainly want to go back and see what people were saying +during their specific presentation. + +Likewise, organizers want to know which presentations people liked and which +ones people didn't enjoy. This could easily influence who they choose to +invite to future conferences. + +For this group the most important aspect of AFA is the organization of the +conversation into chunks, each of which applies to a single presentation. By +reading through each chunk of conversation they can get an idea of the general +response to each presentation. + +[Twitter]: http://twitter.com/ + +Goals +----- + +Once we identified the main users of AFA we were able to come up with the goals +for the new version of the site: + +* **Stay sane while developing.** Ali, Nick and myself are rewriting the site + as a side project, so we don't want it to take *too* much of our time or cause + *too* much stress. + +* **Provide the real time conversation.** The site needs to be fast and + responsive, so people at the conferences can use it to converse. + +* **Don't miss anything.** People following along from home shouldn't feel + like they're missing anything. We want to provide a *complete* version of the + conversation happening behind the event. + +* **Organize the conversation.** Presenters and speakers need a way to see + what people are saying about them. They want to know what people think about + each "chunk" of the event. + +* **Grease the wheels of (physical) social interaction.** This is something + that AFA hasn't tried to address before, but that we'd like to work on with + this version. There are a lot of people at each conference and we'd like to + help them get together. Whether it's going out for dinner or meeting up at the + [Media Temple][mt] party we want to get people talking to each other. I won't + talk about this goal in this post because we're still figuring out the best way + to do it. + +[mt]: http://mediatemple.net/ + +Staying Sane +------------ + +If the three of us tried to create the site with nothing more than a couple of +laptops and a few chats in person we'd go crazy. We use a few tools to help us +manage the development of the site. + +To create **wireframes of the design** we're using a free account at [Hot +Gloo][hotgloo]. Hot Gloo is a great tool that lets us quickly sketch out ideas +and comment on them. + +To share **design comps** we're using [Dropbox][]. It's simple to set up +a shared Dropbox folder and Nick and I can get real time updates when Ali makes +changes to the design. + +To **work together on the code** Nick and I use [Mercurial][] repositories. +Mercurial lets us work on the same code bases simultaneously and we almost +never have to worry about merging. We use [Codebase][] for hosting and issue +tracking. + +[hotgloo]: http://hotgloo.com/ +[Dropbox]: http://dropbox.com/ +[Mercurial]: http://hg-scm.org/ +[Codebase]: http://codebasehq.com/ + +Being Real Time +--------------- + +The previous version of AFA wasn't *truly* real time. When you went to the +site your browser would ask AFA for the newest updates every 10 seconds. + +There are two main problems with this approach. The first is that it's not +really *real time*. I've noticed this being an issue in my own experience. + +When I watch any of the various "live streams" of [Apple][] press conferences +I'm usually at work where other people are also watching. We very rarely load +the pages of the streaming sites like Gizmodo at the exact same second, so our +browsers will be out of sync with each other. Nick might get an update that +I would have to wait 8 more seconds to see. + +When I glance over at his screen and see an update that I don't have +I instinctively refresh the page to get it. This defeats the purpose of the +"live updating" code that the developers of these sites worked on. They may as +well have just made a static page and told me to refresh. + +The second problem is that querying every 10 seconds can be taxing on the +site's database. We're doing this as a side project so we don't have unlimited +funding for a hefty database server. If 1,000 people are querying for updates +every 10 seconds that's 100 requests per second to the database. This means we +need to have some kind of in-memory cacheing if we want the site to feel snappy +on modest hardware. + +We want the new AFA to be truly real time. To do this we need to use [long +polling][longpolling] by users' browsers to wait for updates and return them as +soon as they come in. We also need to retrieve the updates as fast as we +possibly can, and they need to be stored in memory to avoid hitting the +database constantly. + +[Apple]: http://apple.com/ +[longpolling]: http://en.wikipedia.org/wiki/Push_technology#Long_polling + +### Retrieving Updates + +The bulk of the conversation about AEA conferences comes from Twitter. Tweets +are the most important items that we need to display on the site, so we're +using Twitter's streaming API to pull them in. + +Since we don't want to tie up an entire server to pull in tweets I've decided +to create a [Diesel][]-based application called **The Nozzletron** to parse the +streaming API. Diesel is a [Python][] framework that takes an elegant approach +to asynchronous communication with clients and servers. + +Twitter's streaming API accepts HTTP requests and returns "chunked" responses, +each of which is a tweet. Unfortunately the Diesel's built-in HTTP client +doesn't handle chunked HTTP responses so I had to write some code to handle +them myself. + +The Nozzletron will connect to Twitter's streaming API and wait for data to +come in. If there's no data to process it will relinquish the server's +processor so it can do other things. Processing a single tweet that comes in +doesn't take much time, so the server is free to do other things most of the +time. + +I've also created another application called **The Flickrtron** to pull in +[Flickr][] photos. Unfortunately Flickr doesn't have a streaming API like +Twitter so I have to resort to polling Flickr's API every few minutes for new +photos. Flickr is much less of a real time medium than Twitter though, so +I don't think this is a very big problem. + +I'm using a Python library called [Beej's Flickr API][flickrapi] to talk to +Flickr. It is horrible. It calls itself an "API" but is really just a thin +wrapper around calls to the Flickr API. The objects it returns for API calls +are Elementtree objects representing the XML of the response. + +I wish I could do something like: + +```python +thumb_url = photo.thumbnail_url +``` + +Instead I have to use this monstrosity: + +```python +thumb_url = "http://farm%s.static.flickr.com/%s/%s_%s_s.jpg" % ( + photo.get('farm'), photo.get('server'), photo.get('id'), + photo.get('secret'), +) +``` + +I wish there were a better Python Flickr API out there but there doesn't seem +to be one. If I've missed it please let me know! + +[Diesel]: http://dieselweb.org/ +[Python]: http://python.org/ +[Flickr]: http://flickr.com/ +[flickrapi]: http://stuvel.eu/projects/flickrapi + +### Storing Updates + +We need a fast way to store and retrieve updates so AFA can provide a real time +view of the conversation happening at AEA. With this in mind I've chosen +[Redis][] to store the updates of the currently-happening event. + +Redis is an easy-to-set-up, disgustingly-fast, in-memory data store. It's +similar to [memcached][] but has more intelligent data structures that make my +life easier for this project. + +As updates (tweets and flickr photos) are scraped by The Nozzletron and The +Flickrtron they're placed at the tail end of a Redis list. That means I can +quickly and easily get all the items since item `N` when a user's browser +requests them with a single `LRANGE {list-key} N -1` call. + +I'm also keeping a few other pieces of information in Redis. For example, +there's a set of photo IDs held in the `{item-key}:flickrtron:grabbed_photos` +key that keeps track of all of the photos we've already seen. + +This makes it easy to tell if we've already seen a photo (and therefore don't +need to query Flickr for more information) — it's a simple `SISMEMBER +{item-key}:flickrtron:grabbed_photos {photo-id}` call. + +I'm also using Redis to store statistics about the site like: + +* How many tweets we've scraped. +* How many photos we've scraped. +* How many people are currently waiting for updates. + +This kind of information will be extremely valuable in the future when we're +planning improvements to the site. Redis makes it fast and safe to update this +information using the `INCRBY` and `DECR` commands. + +There's one more component to The Nozzletron and The Flickrton that I haven't +mentioned. Both use Redis' `PUBLISH` command to push new updates out to users' +browsers as they arrive. I'll talk more about that in the next section. + +[Redis]: http://code.google.com/p/redis/ +[memcached]: http://memcached.org/ + +### Sending Updates to Users + +As I mentioned before we want to send updates to users *as soon as they're +received*. To do this I've created another Diesel-based application called +**Halley** to handle this [Comet][]-style communication. + +Halley has a few components. The first uses Diesel's [Redis API][dieselredis] +to subscribe to a Redis channel like `live:{event-id}:items` and +[fire][dieselfire] off messages whenever something new comes in. As soon as +a new update comes in from The Nozzletron or The Flickrtron all of Halley's +clients will get it. + +When I started working on AFA Diesel's Redis client didn't support the very new +`PUBLISH`/`SUBSCRIBE`/`UNSUBSCRIBE` commands. I implemented them, put my +changes on [BitBucket][], sent a pull request, and started talking to the +Diesel crew on IRC. + +One of the maintainers pulled my patches and made them even better. Now +Diesel's Redis client has full `PUBLISH`/`SUBSCRIBE`/`UNSUBSCRIBE` support. +It's a great example of how open source projects can produce awesome results. + +The other main component of Halley is an HTTP server that listens for +connections from browsers. The Javascript on the site will call Halley and say +"I need updates since number `X`", where `X` is the length of the Redis list of +updates at the last time it spoke to the server. + +Halley uses Diesel's [HTTP Server][dieselhttp] to manage these requests. + +If a client is asking for everything since `X` and `X` is a smaller number than +the current number of items it will return the updates that have happened since +then. This might happen if we return an update and then two more updates +happen before the browser gets around to sending another request. + +If a client is asking for everything since `X` and `X` is equal to the current +number of items Halley will wait for a new message to be fired from the +[Loop][dieselloop] that's watching the Redis channel. While Halley waits she +relinquishes the processor to the server so other requests can be handled. + +There's a bit of code to prevent DoS attacks that request every item in the +queue over and over again, of course. + +[Comet]: http://en.wikipedia.org/wiki/Comet_(programming) +[dieselredis]: http://bitbucket.org/boomplex/diesel/src/tip/diesel/protocols/redis.py +[dieselfire]: http://bitbucket.org/boomplex/diesel/src/tip/diesel/core.py#cl-90 +[BitBucket]: http://bitbucket.org/ +[dieselhttp]: http://bitbucket.org/boomplex/diesel/src/tip/diesel/protocols/http.py#cl-156 +[dieselloop]: http://bitbucket.org/boomplex/diesel/src/tip/diesel/core.py#cl-179 + +Organizing the Conversation +--------------------------- + +The live stream is an important component of AFA, but it's not the only one. +We also need to organize updates into logical chunks by event and presentation, +and provide archives of old events so people can see what happened. + +The main AFA site is built with [Django][] and served with [Gunicorn][] and +[Nginx][]. It uses a [Postgresql][] database to store data that's not "live". +Because queries for live data are handled with Diesel and Redis we don't need +to send those request through the full Django/Postgresql stack. Django and +Postgresql are only involved when you load a fresh page, and they're *more* +than capable of handling the amount of traffic that AFA gets for those kind of +requests. + +I've created an application called **The Strainer** to copy data from the live +stream to the Postgresql database. The Strainer looks at the list of live +items in Redis, parses those items into Django models and saves them to the +Postgresql database. + +Using two different types of stores (Redis and Postgresql) means we can get the +best of both worlds for AFA: + +* We can keep the live data that's accessed *constantly* in an in-memory Redis + datastore which makes it blazingly fast. + +* We can keep the less-frequently-used data in an on-disk Postgresql database + which lets us to keep our memory usage low and hosting costs down. + +Django's models and managers make it very easy to separate updates out into the +various sessions and presentations that happen at the conferences. + +Sessions and presentations are much less in-demand than the live stream so we +can take advantage of Django's abstractions without worrying about the extra +memory/CPU usage we incur by doing so. + +[Django]: http://djangoproject.com/ +[Gunicorn]: http://gunicorn.org/ +[Nginx]: http://nginx.org/ +[Postgresql]: http://www.postgresql.org/ + +Staying Consistent +------------------ + +Another problem AFA has faced in the past is losing tweets. No code is perfect +(and mine *certainly* is not) so we need to anticipate that some of the +applications we're using will crash at some point. + +I'm using [Supervisord][] to monitor the various processes of AFA on the +server. If a process crashes for some reason it will be restarted +automatically. + +Supervisord also has a wonderful Python API, so I've created a simple Dashboard +view in the Django site that lets us stop/start/restart each individual process +with a simple web interface. The dashboard also shows us the current memory +usage of the server and some other statistics so we can monitor how things are +working through a web browser (instead of SSH'ing into the server, which is +a pain on a phone). + +[Supervisord]: http://supervisord.org/ + +Getting Bigger +-------------- + +An Event Apart is a large event, but it's not a *huge* event. Despite this +I've been trying to build the backend in a way that can be easily scaled. + +Right now it's hosted on a single server, but each of the individual components +could be moved to a separate server with less than an hour of work each: + +* Redis +* Postgresql +* Django, Nginx and Gunicorn +* The Nozzletron +* The Flickrtron +* The Strainer +* Halley + +Moving Django/Gunicorn, Nginx, Redis, Halley, and Postgresql to dedicated +servers would increase the performance of the site *immensely*. I can't +imagine an event that would provide enough traffic to require more than that. + +Even if there were an event that needed that kind of throughput, we could +easily split the single Halley and Redis servers into multiple load-balanced +servers. + +A Work in Progress +------------------ + +This new version of A Feed Apart is still being built. I'm learning new things +every time I work on it, and I'm sure there's still room for improvement. + +If you have any questions, advice, or want me to go more in-depth about +a specific aspect of the site's backend please let me know! + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2010/05/mercurial-workflows-stable-default.html --- a/content/blog/2010/05/mercurial-workflows-stable-default.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,163 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "Mercurial Workflows: Stable & Default" - snip: "Part 2 of several." - created: 2010-05-17 18:27:00 -%} - -{% block article %} - -This entry is the second in my series describing various Mercurial workflows. -The [first][branch-as-needed] describes the simplest one: branching only -when necessary. - -If you're working on a larger project you might want something with a bit more -structure. This post is about the "stable and default" workflow. - -[branch-as-needed]: /blog/2010/02/mercurial-workflows-branch-as-needed/ - -[TOC] - -"Stable and Default" in a Nutshell ----------------------------------- - -The general idea of this workflow is that you keep two branches: `default` and -`stable`. - -* **`default`** is the branch where new features and functionality are added. -* **`stable`** is where bug fixes are added, as well as documentation improvements - that don't pertain to new features. - -Each time you make a bug fix in `stable` you merge it into `default`, so -`default` is always a superset of `stable`. - -Periodically (whenever you're ready for a "major release") you'll merge -`default` into `stable` so new features can be included in releases. - -[Mercurial][] itself [uses][hg-branches] this workflow for development, so it -can scale well to projects of moderate to large size. - -[Mercurial]: http://hg-scm.org/ -[hg-branches]: http://selenic.com/repo/hg/branches/ - -Branch Setup ------------- - -To get started using this workflow you'll need to create a `stable` named -branch: - - :::console - hg branch stable - hg commit -m "Create the stable branch." - -Once you do this users of your project can clone the `stable` branch and be -confident that they're getting a relatively stable version of your code. To -clone a branch like this they would do something like: - - :::console - hg clone http://bitbucket.org/you/yourproject#stable - -This will clone your project's repository and include only changesets on the -`stable` branch (and any of their ancestors). - -Making Changes --------------- - -The goal of this workflow is to do all non-bugfix development on the `default` -branch. Pure bug fixes should go on the `stable` branch so `stable` stays as, -well, "stable" as possible. - -Users that want to live on the bleeding edge of development can use the -`default` branch of your project. Hopefully your project has some users that -are willing to work with `default` and inform you of bugs found with the new -functionality you add to it. - -Whenever you make a change to `stable` you'll want to merge it into `default` -so that `default` always remains a superset of `stable`. This makes `default` -as stable as it can possibly be. It also makes it easier to merge `default` -back into stable whenever you're ready for a major release. - -Here's an example of how your repository's graph will end up looking: - -![Sample Default and Stable Graph](/media/images{{parent_url}}/default-stable-example.png "Sample Default and Stable Graph") - -Notice how each time some changes are made on `stable` they're merged to -`default`. - -Releasing Major Versions ------------------------- - -There will come a time when you're ready to release non-bugfix improvements to -your project to the general public. Non-bugfix improvements are made in the -`default` branch, so when you're ready to do this you'll merge `default` into -`stable`. - -Because your project has more `stable` users than bleeding-edge users, you'll -probably get more bug reports than usual after you release a major version. -This is to be expected and you should be ready for it. - -Tagging Releases ----------------- - -Any decent project should tag releases. This lets users easily use a version -of your project that they know works. - -Wondering how to decide when to tag releases, and what to use for the tags? -The [semantic versioning][semver] specification is a great guide that makes it -easy for your users to know (in a broad sense) what each release changes. - -In a nutshell, tags in a semantically versioned project work like this: - -* Tags are of the form "v[MAJOR].[MINOR].[BUGFIX]" -* Tags with a major version of "0" make no guarantees about anything. They are - used for alpha/beta versions of the project. -* An increase in the bugfix version of a project means "bugs were fixed." -* An increase in the minor version of a project means "functionality has been - added without breaking backwards compatibility." -* An increase in the major version of a project means "backwards compatibility - has been broken." - -Unfortunately this workflow makes it a bit more complicated to add semantic -versioning tags to your project. The rules for semantic tagging would work like -this: - -* When you fix a bug on the `stable` branch, increment the bugfix version on - `stable` and merge `stable` into `default`. -* When you add new functionality and are ready to release it to the public, - merge `default` into stable and increment the minor version of `stable`. -* When you're ready for a backwards-incompatible release, merge `default` into - stable and increment the major version of `stable`. - -The problem with this is that `default` never has any version tags. However, -this probably isn't a big deal because users of `default` are those that want -to live on the bleeding edge of your project and aren't as concerned with -stability. - -[semver]: http://semver.org/ - -Why Default and Stable Instead of Default and Dev? --------------------------------------------------- - -In the workflow I've described there are two branches: `default` and `stable`. -You might be wondering why `default` is used for new development and the -"stable" branch is relegated to a named branch. - -The reason is that `default` will typically have many, many more changesets -added to it than `stable`, and so making the "development" branch the default -makes it easier on the developers. - -There is absolutely *nothing* wrong with making `default` the "stable" branch -and creating a `dev` branch for "unstable" changes. If your project rarely adds -new functionality but is more concerned with fixing bugs this version of the -workflow will obviously be better for you. - -This version also has the added advantage of giving users that naively clone -your project (without a branch specified) the stable version. Since many users -don't bother to read instructions even when you provide them, there is a strong -argument for using it even when your project is *not* overly concerned with bug -fixes. - -It's up to you to decide which version you want to use. - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2010/05/mercurial-workflows-stable-default.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2010/05/mercurial-workflows-stable-default.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,162 @@ ++++ +title = "Mercurial Workflows: Stable & Default" +snip = "Part 2 of several." +date = 2010-05-17T18:27:00Z +draft = false + ++++ + +This entry is the second in my series describing various Mercurial workflows. +The [first][branch-as-needed] describes the simplest one: branching only +when necessary. + +If you're working on a larger project you might want something with a bit more +structure. This post is about the "stable and default" workflow. + +[branch-as-needed]: /blog/2010/02/mercurial-workflows-branch-as-needed/ + +{{% toc %}} + +"Stable and Default" in a Nutshell +---------------------------------- + +The general idea of this workflow is that you keep two branches: `default` and +`stable`. + +* **`default`** is the branch where new features and functionality are added. +* **`stable`** is where bug fixes are added, as well as documentation improvements + that don't pertain to new features. + +Each time you make a bug fix in `stable` you merge it into `default`, so +`default` is always a superset of `stable`. + +Periodically (whenever you're ready for a "major release") you'll merge +`default` into `stable` so new features can be included in releases. + +[Mercurial][] itself [uses][hg-branches] this workflow for development, so it +can scale well to projects of moderate to large size. + +[Mercurial]: http://hg-scm.org/ +[hg-branches]: http://selenic.com/repo/hg/branches/ + +Branch Setup +------------ + +To get started using this workflow you'll need to create a `stable` named +branch: + +```console +hg branch stable +hg commit -m "Create the stable branch." +``` + +Once you do this users of your project can clone the `stable` branch and be +confident that they're getting a relatively stable version of your code. To +clone a branch like this they would do something like: + +```console +hg clone http://bitbucket.org/you/yourproject#stable +``` + +This will clone your project's repository and include only changesets on the +`stable` branch (and any of their ancestors). + +Making Changes +-------------- + +The goal of this workflow is to do all non-bugfix development on the `default` +branch. Pure bug fixes should go on the `stable` branch so `stable` stays as, +well, "stable" as possible. + +Users that want to live on the bleeding edge of development can use the +`default` branch of your project. Hopefully your project has some users that +are willing to work with `default` and inform you of bugs found with the new +functionality you add to it. + +Whenever you make a change to `stable` you'll want to merge it into `default` +so that `default` always remains a superset of `stable`. This makes `default` +as stable as it can possibly be. It also makes it easier to merge `default` +back into stable whenever you're ready for a major release. + +Here's an example of how your repository's graph will end up looking: + +![Sample Default and Stable Graph](/media/images/blog/2010/05/default-stable-example.png "Sample Default and Stable Graph") + +Notice how each time some changes are made on `stable` they're merged to +`default`. + +Releasing Major Versions +------------------------ + +There will come a time when you're ready to release non-bugfix improvements to +your project to the general public. Non-bugfix improvements are made in the +`default` branch, so when you're ready to do this you'll merge `default` into +`stable`. + +Because your project has more `stable` users than bleeding-edge users, you'll +probably get more bug reports than usual after you release a major version. +This is to be expected and you should be ready for it. + +Tagging Releases +---------------- + +Any decent project should tag releases. This lets users easily use a version +of your project that they know works. + +Wondering how to decide when to tag releases, and what to use for the tags? +The [semantic versioning][semver] specification is a great guide that makes it +easy for your users to know (in a broad sense) what each release changes. + +In a nutshell, tags in a semantically versioned project work like this: + +* Tags are of the form "v[MAJOR].[MINOR].[BUGFIX]" +* Tags with a major version of "0" make no guarantees about anything. They are + used for alpha/beta versions of the project. +* An increase in the bugfix version of a project means "bugs were fixed." +* An increase in the minor version of a project means "functionality has been + added without breaking backwards compatibility." +* An increase in the major version of a project means "backwards compatibility + has been broken." + +Unfortunately this workflow makes it a bit more complicated to add semantic +versioning tags to your project. The rules for semantic tagging would work like +this: + +* When you fix a bug on the `stable` branch, increment the bugfix version on + `stable` and merge `stable` into `default`. +* When you add new functionality and are ready to release it to the public, + merge `default` into stable and increment the minor version of `stable`. +* When you're ready for a backwards-incompatible release, merge `default` into + stable and increment the major version of `stable`. + +The problem with this is that `default` never has any version tags. However, +this probably isn't a big deal because users of `default` are those that want +to live on the bleeding edge of your project and aren't as concerned with +stability. + +[semver]: http://semver.org/ + +Why Default and Stable Instead of Default and Dev? +-------------------------------------------------- + +In the workflow I've described there are two branches: `default` and `stable`. +You might be wondering why `default` is used for new development and the +"stable" branch is relegated to a named branch. + +The reason is that `default` will typically have many, many more changesets +added to it than `stable`, and so making the "development" branch the default +makes it easier on the developers. + +There is absolutely *nothing* wrong with making `default` the "stable" branch +and creating a `dev` branch for "unstable" changes. If your project rarely adds +new functionality but is more concerned with fixing bugs this version of the +workflow will obviously be better for you. + +This version also has the added advantage of giving users that naively clone +your project (without a branch specified) the stable version. Since many users +don't bother to read instructions even when you provide them, there is a strong +argument for using it even when your project is *not* overly concerned with bug +fixes. + +It's up to you to decide which version you want to use. + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2010/06/mercurial-workflows-translation-branches.html --- a/content/blog/2010/06/mercurial-workflows-translation-branches.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,122 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "Mercurial Workflows: Translation Branches" - snip: "Uncommon but useful." - created: 2010-06-11 08:15:00 -%} - -{% block article_class %}with-diagrams{% endblock %} - -{% block article %} - -This entry is the third in my series describing various [Mercurial][] workflows. -The [first][branch-as-needed] describes the simplest one: branching only when -necessary, and the [second][default-and-stable] describes the classic "default -and stable" workflow. - -I've been experimenting with another workflow lately and it's proven itself to -be pretty useful, so I wanted to write about it. - -[Mercurial]: {{ links.mercurial }} -[branch-as-needed]: /blog/2010/02/mercurial-workflows-branch-as-needed/ -[default-and-stable]: /blog/2010/05/mercurial-workflows-stable-default/ - -[TOC] - -A Real Example --------------- - -I created the [hgtip.com][] site a while back as a resource for all those tiny -little tips that make working with Mercurial easier. People seemed to like it -and eventually there was some interest in translating the site to other -languages. - -The site itself is a collection of flat files that are run through [Hyde][] to -generate a set of static HTML pages. The content of the site is held in -a [subrepo][] which people can clone to contribute without worrying about the -site's structure. - -When coming up with a strategy for managing translations I had the following -requirements: - -* Translations need to be version controlled. Version control is extremely - helpful, so we need to use it. -* Contributing translations should be (almost) as easy as contributing to the - main site. I didn't want it to be much more work than a simple clone, commit, - push. -* I make typos when creating tips, and sometimes people comment with ideas - I hadn't thought of, which I then add to the tips. When the main (English) - version of a tip is updated it needs to be easy for translators to see - exactly what changed so they can update their translations. - -After thinking about the problem for a while I settled on using Mercurial's -named branches to manage translations. - -[hgtip.com]: http://hgtip.com/ -[Hyde]: {{ links.hyde }} -[subrepo]: http://mercurial.selenic.com/wiki/subrepos - - -Branch Structure ----------------- - -For the content of hgtip I (currently) have three branches: `default`, `ja`, -and `de`. - -`default` contains the English version of the tips, while `ja` and `de` contain -the Japanese and German translations. - -As other people translate the existing tips they commit to the appropriate -branch. When I publish the site to the server I can easily update to each -branch to render the content, one at a time. - -When I add new tips I commit them to the `default` branch and merge `default` -into all the translation branches. This adds the new file to the translation -branches so it gets rendered (in English) on the other versions of the site. - -When a translator is ready to translate any new tips, they can run `hg log -b -THEIRBRANCH` to see what's been merged into the branch and find the tips they -need to translate. - -Here's a diagram to help explain what's going on. As new tips are added they -get merged into the translation branches. When a translator has time to -translate a tip, they commit to their branch. - -![Translation Branching Diagram](/media/images{{ parent_url }}/translation-branches.png "Translation Branches") - - -Linebreaks ----------- - -There's one small issue that arises with a workflow like this, and it happens -when I fix a typo (or add an update) in an existing tip. - -If I fix a typo in `beginner/sometip.html`, it will cause a merge conflict when -someone tries to merge it into a translation branch. This is generally a good -thing because it shows the translator where the change was. They can then fix -the conflict by changing the translation, commit, and everything works. - -The tricky part is that Mercurial will only show line-by-line diffs. If each -paragraph of a tip is a single line, Mercurial will only tell you the paragraph -that changed. For large paragraphs this is annoying because you need to figure -out by hand exactly which word was modified. - -To avoid this I always make sure that lines are hard-wrapped with linebreaks. -You can do this easily by pressing `Ctrl-Q` in TextMate or using `gqip` in vim. -I'm sure Emacs has an equivalent as well. - -This doesn't affect the output because hgtip uses [Markdown][] and Markdown -ignores single linebreaks. If you're using another markup language this might -be a bigger problem for you than it is for me. - -[Markdown]: {{ links.markdown }} - -A Work in Progress ------------------- - -This is the first time I've ever used this branching structure and I haven't -hear about other people using it. I'm sure there are some tricks that might -make things smoother, so if you have any advice I'd love to hear it! - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2010/06/mercurial-workflows-translation-branches.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2010/06/mercurial-workflows-translation-branches.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,118 @@ ++++ +title = "Mercurial Workflows: Translation Branches" +snip = "Uncommon but useful." +date = 2010-06-11T08:15:00Z +draft = false + ++++ + +This entry is the third in my series describing various [Mercurial][] workflows. +The [first][branch-as-needed] describes the simplest one: branching only when +necessary, and the [second][default-and-stable] describes the classic "default +and stable" workflow. + +I've been experimenting with another workflow lately and it's proven itself to +be pretty useful, so I wanted to write about it. + +[Mercurial]: {{ links.mercurial }} +[branch-as-needed]: /blog/2010/02/mercurial-workflows-branch-as-needed/ +[default-and-stable]: /blog/2010/05/mercurial-workflows-stable-default/ + +{{% toc %}} + +A Real Example +-------------- + +I created the [hgtip.com][] site a while back as a resource for all those tiny +little tips that make working with Mercurial easier. People seemed to like it +and eventually there was some interest in translating the site to other +languages. + +The site itself is a collection of flat files that are run through [Hyde][] to +generate a set of static HTML pages. The content of the site is held in +a [subrepo][] which people can clone to contribute without worrying about the +site's structure. + +When coming up with a strategy for managing translations I had the following +requirements: + +* Translations need to be version controlled. Version control is extremely + helpful, so we need to use it. +* Contributing translations should be (almost) as easy as contributing to the + main site. I didn't want it to be much more work than a simple clone, commit, + push. +* I make typos when creating tips, and sometimes people comment with ideas + I hadn't thought of, which I then add to the tips. When the main (English) + version of a tip is updated it needs to be easy for translators to see + exactly what changed so they can update their translations. + +After thinking about the problem for a while I settled on using Mercurial's +named branches to manage translations. + +[hgtip.com]: http://hgtip.com/ +[Hyde]: {{ links.hyde }} +[subrepo]: http://mercurial.selenic.com/wiki/subrepos + + +Branch Structure +---------------- + +For the content of hgtip I (currently) have three branches: `default`, `ja`, +and `de`. + +`default` contains the English version of the tips, while `ja` and `de` contain +the Japanese and German translations. + +As other people translate the existing tips they commit to the appropriate +branch. When I publish the site to the server I can easily update to each +branch to render the content, one at a time. + +When I add new tips I commit them to the `default` branch and merge `default` +into all the translation branches. This adds the new file to the translation +branches so it gets rendered (in English) on the other versions of the site. + +When a translator is ready to translate any new tips, they can run `hg log -b +THEIRBRANCH` to see what's been merged into the branch and find the tips they +need to translate. + +Here's a diagram to help explain what's going on. As new tips are added they +get merged into the translation branches. When a translator has time to +translate a tip, they commit to their branch. + +Translation Branching Diagram + +Linebreaks +---------- + +There's one small issue that arises with a workflow like this, and it happens +when I fix a typo (or add an update) in an existing tip. + +If I fix a typo in `beginner/sometip.html`, it will cause a merge conflict when +someone tries to merge it into a translation branch. This is generally a good +thing because it shows the translator where the change was. They can then fix +the conflict by changing the translation, commit, and everything works. + +The tricky part is that Mercurial will only show line-by-line diffs. If each +paragraph of a tip is a single line, Mercurial will only tell you the paragraph +that changed. For large paragraphs this is annoying because you need to figure +out by hand exactly which word was modified. + +To avoid this I always make sure that lines are hard-wrapped with linebreaks. +You can do this easily by pressing `Ctrl-Q` in TextMate or using `gqip` in vim. +I'm sure Emacs has an equivalent as well. + +This doesn't affect the output because hgtip uses [Markdown][] and Markdown +ignores single linebreaks. If you're using another markup language this might +be a bigger problem for you than it is for me. + +[Markdown]: {{ links.markdown }} + +A Work in Progress +------------------ + +This is the first time I've ever used this branching structure and I haven't +hear about other people using it. I'm sure there are some tricks that might +make things smoother, so if you have any advice I'd love to hear it! + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2010/08/a-git-users-guide-to-mercurial-queues.html --- a/content/blog/2010/08/a-git-users-guide-to-mercurial-queues.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,267 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "A Git User's Guide to Mercurial Queues" - snip: "MQ is git's index on steroids." - created: 2010-08-10 21:00:00 -%} - -{% block article_class %}with-diagrams{% endblock %} - -{% block article %} - -I've been using [Mercurial Queues][MQ] more and more lately. At the last -Mercurial sprint [Brendan Cully][brendan] said something that made me realize -that MQ behaves very much like a souped-up version of [git][]'s index. - -I wanted to write a blog post about the similarities between the two concepts -so that git users could understand [Mercurial][]'s MQ extension a bit better -and see how it can take that concept even further than git does. - -This post is *not* intended to be a guide to MQ's day-to-day commands -- it's -simply trying to explain MQ in a way that git users might find more -understandable. For a primer on MQ commands you can check out [the MQ chapter -in the hg book][mq-book]. - -[MQ]: http://mercurial.selenic.com/wiki/MqExtension -[brendan]: http://cs.ubc.ca/~brendan/ -[git]: {{links.git}} -[Mercurial]: {{links.mercurial}} -[mq-book]: http://hgbook.red-bean.com/read/managing-change-with-mercurial-queues.html - -[TOC] - -Git Basics ----------- - -Let's take a few moments to review how git works so we're all on the same page -with our terminology. - -When you're working with a git repository you have three "layers" to work with: - -* The working directory -* The index -* The git repository - -You use `git add` to shove changes from the working directory into the index -and `git commit` to shove changes from the index into the repository: - -![Git Basics Diagram](/media/images{{ parent_url }}/git-basics.png "Git Basics") - -This is a very powerful model because it lets you build your changesets -piece-by-piece and commit them permanently only when you're ready. - -Mercurial Basics ----------------- - -With basic, stock Mercurial you only have two "layers" to work with: - -* The working directory -* The Mercurial repository - -You use `hg commit` to shove changes from the working directory into the -repository: - -![Mercurial Basics Diagram](/media/images{{ parent_url }}/mercurial-basics.png "Mercurial Basics") - -This model doesn't give you as much flexibility in creating changesets as -git's does. You can use the [record extension][record] to get closer, but it's still -not the same. - -[record]: http://mercurial.selenic.com/wiki/RecordExtension - -Let's take a look at MQ to see how it can give us everything git's index does -*and more.* - -Using MQ with a Single Patch ----------------------------- - -The most basic way to use MQ is to create a single patch with `hg qnew NAME`. -You can make changes in your working directory and use `hg qrefresh` (or `hg -qrecord`) to put them into the patch. Once you're done with your patch and -ready for it to become a commit you can run `hg qfinish`: - -![MQ with One Patch](/media/images{{ parent_url }}/mq-one.png "MQ with One Patch") - -This looks a lot like the diagram of how git works, doesn't it? MQ gives you an -"intermediate" area to put changes, similar to how git's index works. - -Using MQ with Two (or More) Patches ------------------------------------ - -This single "intermediate" area is where git stops. For many workflows it's -enough, but if you want more power MQ has you covered. - -MQ is called Mercurial *Queues* for a reason. You can have more than one patch -in your queue, which means you can have multiple "intermediate" areas if you -need them. - -For example: say you're adding a feature that requires some API changes to your -project. You'd like to commit the changes to the API in one changeset, and the -changes to the interface in another changeset. You can do this by creating two -patches with `hg qnew api-changes; hg qnew interface-changes`: - -![MQ with Two Patches](/media/images{{ parent_url }}/mq-two.png "MQ with Two Patches") - -You can move back and forth between these patches with `hg qpop` and `hg -qpush`. If you're working on the interface and realize you forgot to make -a necessary change to the API you can: - -* `hg qpop` the `interface-changes` patch to get to the `api-changes` patch. -* Make your API changes. -* `hg qrefresh` to put those changes into the `api-changes` patch. -* `hg qpush` to get back to work on the interface. - -Using multiple patches is like having multiple git indexes to store related -changes until you're ready to commit them permanently. - -Multiple Patch Queues ---------------------- - -What happens when you want to work on two features, each with two patches, at -the same time? You *could* simply create four patches and let the second -feature live on top of the first, but there's a better way. - -Mercurial 1.6 (I think) added the `hg qqueue` command, which lets you create -*multiple* patch queues, each one living in its own directory. That means you -can create a separate queue (with its own set of patches) with `hg qqueue -c -NAME` for each feature: - -![MQ with Multiple Queues](/media/images{{ parent_url }}/mq-multiple.png "MQ with Multiple Queues") - -You can switch patch queues with `hg qqueue NAME`. This gives you multiple -sets of "intermediate" areas like git's index to work with. This is probably -not something you'll need very often, but it's there when you *do* need it. - -You can see that MQ is already quite a bit more flexible than git's index, but -it has one more trick up its sleeve. - -Versioned Patch Queues ----------------------- - -Let me prefix this section by saying: "You might think you need to use versioned -patch queues, but you probably don't." Versioned queues can be tricky to wrap -your head around, but once you understand them you'll realize how powerful they -can be. - -Let's pause a second to look at how MQ actually stores its patches. - -When you create a new patch with `hg qnew interface-changes` Mercurial will -create a `patches` folder inside the `.hg` folder of your project. Your new -patch is stored in a file inside that folder (along with some other metadata -files): - - :::text - yourproject/ - | - +-- .hg/ - | | - | +-- patches/ - | | | - | | +-- api-changes - | | +-- interface-changes - | | `-- ... other MQ-related files ... - | | - | `-- ... other Mercurial-related files ... - | - `-- ... your project's files ... - -When you create a new patch queue with `hg qqueue -c some-feature` Mercurial -creates a completely separate `patches-some-feature` folder in `.hg`: - - :::text - yourproject/ - | - +-- .hg/ - | | - | +-- patches/ - | | | - | | +-- api-changes - | | +-- interface-changes - | | `-- ... other MQ-related files ... - | | - | +-- patches-some-feature/ - | | | - | | +-- api-changes - | | +-- interface-changes - | | `-- ... other MQ-related files ... - | | - | `-- ... other mercurial-related files ... - | - `-- ... your project's files ... - -These folders are normal filesystem folders. The patches inside them are -plain-text files. - -This gives them a very important property: - -**They can be turned into Mercurial repositories to track changes.** - -Once you understand what this means, you should have several "oh my god" -moments where you realize several very interesting things: - -* Not only can you have multiple sets of "intermediate" areas to work with, you - can *version* them to keep track of the changes! -* You can share queues with other people with Mercurial's vanilla push/pull - commands. -* You can collaborate with other people and merge your changes to these - "intermediate" areas with Mercurial's vanilla push/pull/merge commands. -* You can `hg serve` these patch repositories so other people can see what your - patches look like before they've been permanently committed to your project's - repository. - -Versioning patch queues means you can end up with a (hard to read) diagram like -this: - -![Versioned Queues](/media/images{{ parent_url }}/mq-versioned.png "Versioned Queues") - -To facilitate working with versioned patch queues all Mercurial commands come -with a `--mq` option to apply the command to the queue repository instead of -the current one (so you don't need to `cd` to the queue repository all the -time). - -Versioning patch queues is an *incredibly* powerful concept, and most of the -time you won't need it, but it's nice to have it when you do. - -It's also nice to know that [BitBucket][] has [special support][] for version -patch queues. - -[BitBucket]: http://bitbucket.org/ -[special support]: http://ches.nausicaamedia.com/articles/technogeekery/using-mercurial-queues-and-bitbucket-org - -Problems with MQ ----------------- - -Despite how powerful MQ is (or perhaps *because* of it) it has some problems. -Most could be fixed if someone had a week or two to spend on it, but so far no -one has stepped up. - -I'd do it if I could afford to take a week away from full-time/freelance work, -but I can't right now. If you want to be the hero of at least one MQ user (me) -here's what sucks about MQ: - -* There's no easy way to pull changes *out* of an MQ patch. -* There's no easy way to split an MQ patch into two patches (the current - horrible "workaround" is to empty the current patch, `hg qrecord` part one, - and `hg qnew` to make a new patch with part two). -* You can't pop a patch once you've made changes in your working directory. You - need to [shelve][] the changes, pop the patch, and then unshelve the changes. - -[shelve]: http://mercurial.selenic.com/wiki/ShelveExtension - -In addition there's another MQ-related change that would be very, very nice: - -* Refactor the record extension and put it into core Mercurial, so that - `hg qrecord` could be used without an extra extension (and we could have `hg - commit --interactive`). - -If someone takes the time to fix these problems I'll love them forever. Until -then we're stuck with the powerful-but-sometimes-clumsy MQ interface. - -Hopefully this post has given git users (and Mercurial users) an idea of how -powerful MQ can be. If you have any questions please find me on -[Twitter][twsl] and me know! - -[twsl]: {{links.twsl}} - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2010/08/a-git-users-guide-to-mercurial-queues.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2010/08/a-git-users-guide-to-mercurial-queues.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,264 @@ ++++ +title = "A Git User's Guide to Mercurial Queues" +snip = "MQ is git's index on steroids." +date = 2010-08-10T21:00:00Z +draft = false + ++++ + +I've been using [Mercurial Queues][MQ] more and more lately. At the last +Mercurial sprint [Brendan Cully][brendan] said something that made me realize +that MQ behaves very much like a souped-up version of [git][]'s index. + +I wanted to write a blog post about the similarities between the two concepts +so that git users could understand [Mercurial][]'s MQ extension a bit better +and see how it can take that concept even further than git does. + +This post is *not* intended to be a guide to MQ's day-to-day commands — it's +simply trying to explain MQ in a way that git users might find more +understandable. For a primer on MQ commands you can check out [the MQ chapter +in the hg book][mq-book]. + +[MQ]: http://mercurial.selenic.com/wiki/MqExtension +[brendan]: http://cs.ubc.ca/~brendan/ +[git]: {{links.git}} +[Mercurial]: {{links.mercurial}} +[mq-book]: http://hgbook.red-bean.com/read/managing-change-with-mercurial-queues.html + +{{% toc %}} + +Git Basics +---------- + +Let's take a few moments to review how git works so we're all on the same page +with our terminology. + +When you're working with a git repository you have three "layers" to work with: + +* The working directory +* The index +* The git repository + +You use `git add` to shove changes from the working directory into the index +and `git commit` to shove changes from the index into the repository: + +Git Basics + +This is a very powerful model because it lets you build your changesets +piece-by-piece and commit them permanently only when you're ready. + +Mercurial Basics +---------------- + +With basic, stock Mercurial you only have two "layers" to work with: + +* The working directory +* The Mercurial repository + +You use `hg commit` to shove changes from the working directory into the +repository: + +Mercurial Basics + +This model doesn't give you as much flexibility in creating changesets as +git's does. You can use the [record extension][record] to get closer, but it's still +not the same. + +[record]: http://mercurial.selenic.com/wiki/RecordExtension + +Let's take a look at MQ to see how it can give us everything git's index does +*and more.* + +Using MQ with a Single Patch +---------------------------- + +The most basic way to use MQ is to create a single patch with `hg qnew NAME`. +You can make changes in your working directory and use `hg qrefresh` (or `hg +qrecord`) to put them into the patch. Once you're done with your patch and +ready for it to become a commit you can run `hg qfinish`: + +MQ with One Patch + +This looks a lot like the diagram of how git works, doesn't it? MQ gives you an +"intermediate" area to put changes, similar to how git's index works. + +Using MQ with Two (or More) Patches +----------------------------------- + +This single "intermediate" area is where git stops. For many workflows it's +enough, but if you want more power MQ has you covered. + +MQ is called Mercurial *Queues* for a reason. You can have more than one patch +in your queue, which means you can have multiple "intermediate" areas if you +need them. + +For example: say you're adding a feature that requires some API changes to your +project. You'd like to commit the changes to the API in one changeset, and the +changes to the interface in another changeset. You can do this by creating two +patches with `hg qnew api-changes; hg qnew interface-changes`: + +MQ with Two Patches + +You can move back and forth between these patches with `hg qpop` and `hg +qpush`. If you're working on the interface and realize you forgot to make +a necessary change to the API you can: + +* `hg qpop` the `interface-changes` patch to get to the `api-changes` patch. +* Make your API changes. +* `hg qrefresh` to put those changes into the `api-changes` patch. +* `hg qpush` to get back to work on the interface. + +Using multiple patches is like having multiple git indexes to store related +changes until you're ready to commit them permanently. + +Multiple Patch Queues +--------------------- + +What happens when you want to work on two features, each with two patches, at +the same time? You *could* simply create four patches and let the second +feature live on top of the first, but there's a better way. + +Mercurial 1.6 (I think) added the `hg qqueue` command, which lets you create +*multiple* patch queues, each one living in its own directory. That means you +can create a separate queue (with its own set of patches) with `hg qqueue -c +NAME` for each feature: + +MQ with Multiple Queues + +You can switch patch queues with `hg qqueue NAME`. This gives you multiple +sets of "intermediate" areas like git's index to work with. This is probably +not something you'll need very often, but it's there when you *do* need it. + +You can see that MQ is already quite a bit more flexible than git's index, but +it has one more trick up its sleeve. + +Versioned Patch Queues +---------------------- + +Let me prefix this section by saying: "You might think you need to use versioned +patch queues, but you probably don't." Versioned queues can be tricky to wrap +your head around, but once you understand them you'll realize how powerful they +can be. + +Let's pause a second to look at how MQ actually stores its patches. + +When you create a new patch with `hg qnew interface-changes` Mercurial will +create a `patches` folder inside the `.hg` folder of your project. Your new +patch is stored in a file inside that folder (along with some other metadata +files): + +```text +yourproject/ +| ++-- .hg/ +| | +| +-- patches/ +| | | +| | +-- api-changes +| | +-- interface-changes +| | `-- ... other MQ-related files ... +| | +| `-- ... other Mercurial-related files ... +| +`-- ... your project's files ... +``` + +When you create a new patch queue with `hg qqueue -c some-feature` Mercurial +creates a completely separate `patches-some-feature` folder in `.hg`: + +```text +yourproject/ +| ++-- .hg/ +| | +| +-- patches/ +| | | +| | +-- api-changes +| | +-- interface-changes +| | `-- ... other MQ-related files ... +| | +| +-- patches-some-feature/ +| | | +| | +-- api-changes +| | +-- interface-changes +| | `-- ... other MQ-related files ... +| | +| `-- ... other mercurial-related files ... +| +`-- ... your project's files ... +``` + +These folders are normal filesystem folders. The patches inside them are +plain-text files. + +This gives them a very important property: + +**They can be turned into Mercurial repositories to track changes.** + +Once you understand what this means, you should have several "oh my god" +moments where you realize several very interesting things: + +* Not only can you have multiple sets of "intermediate" areas to work with, you + can *version* them to keep track of the changes! +* You can share queues with other people with Mercurial's vanilla push/pull + commands. +* You can collaborate with other people and merge your changes to these + "intermediate" areas with Mercurial's vanilla push/pull/merge commands. +* You can `hg serve` these patch repositories so other people can see what your + patches look like before they've been permanently committed to your project's + repository. + +Versioning patch queues means you can end up with a (hard to read) diagram like +this: + +Versioned Queues + +To facilitate working with versioned patch queues all Mercurial commands come +with a `--mq` option to apply the command to the queue repository instead of +the current one (so you don't need to `cd` to the queue repository all the +time). + +Versioning patch queues is an *incredibly* powerful concept, and most of the +time you won't need it, but it's nice to have it when you do. + +It's also nice to know that [BitBucket][] has [special support][] for version +patch queues. + +[BitBucket]: http://bitbucket.org/ +[special support]: http://ches.nausicaamedia.com/articles/technogeekery/using-mercurial-queues-and-bitbucket-org + +Problems with MQ +---------------- + +Despite how powerful MQ is (or perhaps *because* of it) it has some problems. +Most could be fixed if someone had a week or two to spend on it, but so far no +one has stepped up. + +I'd do it if I could afford to take a week away from full-time/freelance work, +but I can't right now. If you want to be the hero of at least one MQ user (me) +here's what sucks about MQ: + +* There's no easy way to pull changes *out* of an MQ patch. +* There's no easy way to split an MQ patch into two patches (the current + horrible "workaround" is to empty the current patch, `hg qrecord` part one, + and `hg qnew` to make a new patch with part two). +* You can't pop a patch once you've made changes in your working directory. You + need to [shelve][] the changes, pop the patch, and then unshelve the changes. + +[shelve]: http://mercurial.selenic.com/wiki/ShelveExtension + +In addition there's another MQ-related change that would be very, very nice: + +* Refactor the record extension and put it into core Mercurial, so that + `hg qrecord` could be used without an extra extension (and we could have `hg + commit --interactive`). + +If someone takes the time to fix these problems I'll love them forever. Until +then we're stuck with the powerful-but-sometimes-clumsy MQ interface. + +Hopefully this post has given git users (and Mercurial users) an idea of how +powerful MQ can be. If you have any questions please find me on +[Twitter][twsl] and me know! + +[twsl]: {{links.twsl}} + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2010/09/coming-home-to-vim.html --- a/content/blog/2010/09/coming-home-to-vim.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,942 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "Coming Home to Vim" - snip: "I'm sorry I ever left, baby." - created: 2010-09-20 18:15:00 - %} - - {% block article %} - -I'm a programmer. I work with text files for 6-12 hours every weekday so I care -about the text editor I use. If switching to a different editor can increase my -efficiency by even 10% it would save a good chunk of my time and let me get -back to making cool things. - -I don't buy the "you're thinking 90% of the time and only typing 10% of the -time, so your editor doesn't really matter" argument. Even if the premise is -true, the conclusion is wrong. - -If I think for 10 minutes and then start typing, I want the typing to take the -shortest time possible so I can get back to thinking. Any time I spend typing -is an interruption that I want to minimize so I can keep my train of thought. - -I recently started using [Vim][] as my primary editor. As I'm adjusting I'm -finding a lot of [the][arrows] [blog][nutheads] [posts][mind] [people][katz] -[have][nvie] [written][grok] [about][home] Vim very helpful, so I'm hoping this -post will help people too. - -[Vim]: http://www.vim.org/ -[arrows]: http://jeetworks.org/node/89 -[nutheads]: http://www.viemu.com/a-why-vi-vim.html -[mind]: http://wekeroad.com/2010/07/29/vim-is-your-daddy/ -[katz]: http://yehudakatz.com/2010/07/29/everyone-who-tried-to-convince-me-to-use-vim-was-wrong/ -[nvie]: http://nvie.com/posts/how-i-boosted-my-vim/ -[grok]: http://stackoverflow.com/questions/1218390/what-is-your-most-productive-shortcut-with-vim/1220118#1220118 -[home]: http://weblog.jamisbuck.org/2008/10/10/coming-home-to-vim - -[TOC] - -Some Background About Me ------------------------- - -I used Vim exclusively from about 2002 to 2005. I never really learned much -about it and mostly used it as a glorified [nano][]. - -In 2006 I decided it was time for a change. I wanted an editor that "fit" with -OS X. I tried [SubEthaEdit][] for a while and it was kind of cool. The -collaborative editing was more polished than any other version I've seen, -though I didn't have much use for it. - -As cool as SubEthaEdit was, I always felt like there was a better editor out -there for me. Then I found [TextMate][]. - -At the time I thought TextMate was amazing (and it really was). It *looked* -like a Mac application. It was super easy to learn (all the OS X text movement -commands just worked). There were bundles for everything. - -I still used Vim occasionally (like when I had a job and only had a Windows -machine), but for the most part I was a TextMate kid. - -After using TextMate for four years I decided it was time for another change. -Development on TextMate has stalled while TextMate Forever (err, TextMate 2) is -in the works. I started thinking about Vim (and reading blog posts by others -about how they switched back) and decided to give it another try. - -A few months ago I bought [Learning the vi and Vim Editors][] and went -cold-turkey with Vim as my main text editor. I haven't looked back since. - -[nano]: http://en.wikipedia.org/wiki/Nano_(text_editor) -[SubEthaEdit]: http://www.codingmonkeys.de/subethaedit/ -[TextMate]: http://macromates.com/ -[Learning the vi and Vim Editors]: http://amzn.com/059652983X - -Why I Switched to TextMate --------------------------- - -As I mentioned before, there were a few reasons I switched to TextMate in the -first place. - -First of all: it looks and acts like a real OS X application. You can drag -files onto the icon to open them, it supports all the default OS X text -movement commands, and the chrome looks like it belongs on OS X. I don't know -if [MacVim][] was around at the time, but if it was I never found it. - -This matters. The effort required to get started with a new editor is -dramatically reduced if that editor supports all the conventions of the -operating system you use every day. - -It also has a vibrant community of people writing bundles for it, so there was -support for almost anything I wanted to use. - -[MacVim]: http://code.google.com/p/macvim/ - -Why I Came Back to Vim ----------------------- - -I came back to Vim for a number of reasons. - -First of all: I started reading about and desiring features that didn't seem to -be coming to TextMate any time soon. The biggest of those was split windows. -I saw the appeal of split windows right away and now that I use them I can't -imagine not having them. - -Another reason is version control. A while ago I started keeping my -[dotfiles][] in a [Mercurial][] repository which makes it extremely easy to set -up new machines. Vim keeps all of its configuration in two simple places: -a simple `~/.vimrc` file and a `~/.vim` directory full of plain text files. - -Vim's plain text configuration files and plugins are very easy to version -control. They diff well, unlike TextMate's multiple `Bundles` directories and -ugly XML file format. - -Vim's community has been around far longer than TextMate's so there's an even -richer set of plugins, bundles and syntax files available. - -Another reason people give for enjoying Vim is that it runs everywhere. Yes, -it's nice to have my favorite editor available when I SSH into a machine, but -it's not a huge deal if I don't. I almost always edit files on my own machine -and then deploy them with an automated script of some kind, so editing on -a server is very rare for me. - -[dotfiles]: http://bitbucket.org/sjl/dotfiles/ -[Mercurial]: http://hg-scm.org/ - -Core Differences ----------------- - -For me, Vim and TextMate are very different. TextMate relies mainly on -keyboard shortcuts (involving Shift, Ctrl, Cmd and Alt) to do more than simply -edit text. Vim uses a "modal editing" idea to accomplish the same task. - -TextMate's philosophy is much the same as any normal OS X application's, and -I said that acting like the OS reduces the barrier of entry for a text editor, -so why do I prefer Vim's philosophy? - -First I should prefix my answer with this: I'm not averse to a learning curve -for something as important to me as a text editor. Yes, it's awesome if -I don't need to unnecessarily rewire my brain, but if it's going to save me -a lot of time then I'm willing to cut a bit of slack here. - -My problem with TextMate's philosophy can be summed up in one word: -"shadowing". - -If I want to define a new command in a bundle (or if someone else has already -done it), I'm never really sure if I'm "overwriting" some other command that my -fingers know but my brain doesn't really register. - -By the time I try to use the default command it could be days or weeks after -I defined the custom one, and I need to figure out what I did all over again. -It's quite frustrating. - -Vim's "insert" mode means that when I'm editing text I'm using the normal OS -X text movement shortcuts that I know and love. When I want to do something -special I enter normal mode and don't have to worry about shadowed commands -(Vim's leader key also helps with this). - -Vim's "Feeling" ---------------- - -Vim's normal mode has a unique "feeling" that I haven't seen in any other text -editor (although [E][] may [change that][modalE]). I've heard it described in -a number of ways, and I think that while no one analogy fully explains it, -together they give a pretty good description. - -[E]: http://www.e-texteditor.com/ -[modalE]: http://e-texteditor.com/blog/2010/beyond-vi - -### A "Language" of Text Editing - -One way to think about Vim's normal-mode commands is like a language. You have -"verbs" and "nouns". For example: the "change" command (`c`) would be a verb -and the "word" item (`w`) is a noun. You can combine them to form imperative -sentences that talk about what you want to do with your text. - -The wonderful part about this is that whenever you learn a new verb (like -"delete" (`d`)) you can immediately apply it to all the nouns you know, and -vice versa. - -Vim also has "adjectives" like "inside" and "around" (`i` and `a`) that let you -craft sentences like "change inside parenthesis" (`ci(` or `cib`). Once you -learn one of these you can immediately apply it to all the verbs and nouns you -already know. - -### The Physics of Text - -The second way to describe the feeling of Vim is "physics". It's a much less -concrete description but I think it's still useful. - -If I throw a bowling ball into the air it behaves much the same way as if -I throw a turkey sandwich. They might have very different effects when they -land on someone but the act of throwing them is pretty much the same. - -Vim's operators/actions/verbs act the same way. `daw` has a very different -effect than `da{` but they can both be understood with a basic underlying -principle: `da` will "delete around ``". - -### "Programming" Your Editing - -The last way that I hear people talking about Vim's editing is: "it feels like -you're 'programming' your text". - -This analogy is often made by programmers (obviously) and I think it works on -more than one level. - -First: Vim's basic editing commands can be likened to function calls. `daw` -can be thought of a running a function like `delete(type='word', around=True)`. -I don't think this way (I prefer the language and physics analogies) but other -people do. - -Although I don't think of the *basic* Vim commands as programming I do see -some aspects of it in other areas. For example: say I define a leader command -in my `~/.vimrc` file: - - :::vim - nnoremap 1 yypVr= - -I think of this as defining `1` to be a function that performs the -following actions: - -* Yank/copy the current line. -* Paste it below (and move down to the pasted version). -* Select the copied line. -* Replace every character with `=`. - -Another more obvious "programming" aspect of Vim is creating macros. Macros are -like tiny little functions you create to help you when you're editing a file. -You can define them, execute them, and even edit them by pasting, editing and -yanking them back. - -Getting Started ---------------- - -When I switched back to Vim I did a few things that helped make my transition -stick. - -First: I bought [Learning the vi and Vim Editors][] and read it cover to cover. -It helps quite a bit to know the history behind Vim, and the book also teaches -you the basics of Vim's editing commands. - -I also looked through [the Vim subreddit][rvim] to find some good blog posts about -switching. A lot of people have written about it and it was very helpful to -read about their experiences. - -And finally, the most important part: I switched cold-turkey. Vim runs everywhere, so -there's really no excuse for not doing this. Every decent OS has a version of -Vim that makes insert mode "just work" for their OS (like gvim or MacVim), so -the barrier to entry these days is pretty low. - -You can get Vim and start using it right away by staying in insert mode most of -the time. Then you can start learning its real features at your own pace. -Trying to use two different editors at once just slows down the learning -process and makes you less productive in both. - -[rvim]: http://reddit.com/r/vim/ - -Making Vim More Useful ----------------------- - -One of the flaws of Vim is that it takes quite a bit of configuration to make -it behave in a decent way. The defaults are backwards-compatible with vi (i.e. -older than most college students) and not very helpful. - -Here are a few of the things I do to make Vim a bit more palatable. - -**A quick side note:** even if you don't use Vim, you *need* to [remap your -capslock key][capslock] to something useful. Just do it, you'll thank me. - -[capslock]: http://c2.com/cgi/wiki?RemapCapsLock - -### Important .vimrc Lines - -I won't go through every line in [my `~/.vimrc` file][sjlvimrc], but here are some of the -lines that I simply could not live without. - -[sjlvimrc]: http://bitbucket.org/sjl/dotfiles/src/tip/vim/.vimrc - -First, a few lines that you absolutely must have: - - :::vim - filetype off - call pathogen#runtime_append_all_bundles() - filetype plugin indent on - - set nocompatible - - set modelines=0 - -The `filetype` and `call` lines are for loading Pathogen, which is described in -the bundles section. See [Pathogen's docs][pathogendocs] to learn about why the -first `filetype` line is there. - -[pathogendocs]: http://www.vim.org/scripts/script.php?script_id=2332 - -`set nocompatible` gets rid of all the crap that Vim does to be vi compatible. -It's 2010 -- we don't need to be compatible with vi at the expense of -functionality any more. - -The `modelines` bit prevents some [security exploits][] having to do with -modelines in files. I never use modelines so I don't miss any functionality -here. - -[security exploits]: http://lists.alioth.debian.org/pipermail/pkg-vim-maintainers/2007-June/004020.html - -Next I set my tab settings: - - :::vim - set tabstop=4 - set shiftwidth=4 - set softtabstop=4 - set expandtab - -I like all tabs to expand to four spaces. Check out [this Vimcast][vctabs] to -learn more about each of these options. - -[vctabs]: http://vimcasts.org/episodes/tabs-and-spaces/ - -Next are a few options that just make things better: - - :::vim - set encoding=utf-8 - set scrolloff=3 - set autoindent - set showmode - set showcmd - set hidden - set wildmenu - set wildmode=list:longest - set visualbell - set cursorline - set ttyfast - set ruler - set backspace=indent,eol,start - set laststatus=2 - set relativenumber - set undofile - -Each of these lines are basically to make Vim behave in a sane manner. The two -"interesting" ones are the last two, and both deal with features that are -[new in Vim 7.3][vim73]. - -[vim73]: http://groups.google.com/group/vim_announce/browse_thread/thread/66c02efd1523554b - -`relativenumber` changes Vim's line number column to display how far away each -line is from the current one, instead of showing the absolute line number. - -I almost never care what numeric line I'm on in a file (and if I do I can see -it in the status line), so I don't miss the normal line numbers. I *do* care -how far away a particular line might be, because it tells me what number I need -to use with motion commands like `dd`. - -`undofile` tells Vim to create `.un~` files whenever you edit a file. -These files contain undo information so you can undo previous actions even -after you close and reopen a file. - -Next I change the `` key: - - :::vim - let mapleader = "," - -For me `,` is easier to type than `\`. I use leader commands constantly so it's -worth changing. - -The next thing I do is tame searching/moving: - - :::vim - nnoremap / /\v - vnoremap / /\v - set ignorecase - set smartcase - set gdefault - set incsearch - set showmatch - set hlsearch - nnoremap :noh - nnoremap % - vnoremap % - -The first two lines fix Vim's horribly broken default regex "handling" by -automatically inserting a `\v` before any string you search for. This turns -off Vim's crazy default regex characters and makes searches use normal regexes. -I already know Perl/Python compatible regex formatting, why would I want to -learn another scheme? - -`ignorecase` and `smartcase` together make Vim deal with case-sensitive search -intelligently. If you search for an all-lowercase string your search will be -case-insensitive, but if one or more characters is uppercase the search will be -case-sensitive. Most of the time this does what you want. - -`gdefault` applies substitutions globally on lines. For example, instead of -`:%s/foo/bar/g` you just type `:%s/foo/bar/`. This is almost always what you -want (when was the last time you wanted to only replace the first occurrence of -a word on a line?) and if you need the previous behavior you just tack on the -`g` again. - -`incsearch`, `showmatch` and `hlsearch` work together to highlight search -results (as you type). It's really quite handy, as long as you have the next -line as well. - -The `` mapping makes it easy to clear out a search by typing -`,`. This gets rid of the distracting highlighting once I've found what -I'm looking for. - -The last two lines make the tab key match bracket pairs. I use this to move -around all the time and `` is a hell of a lot easier to type than `%`. - -The next section makes Vim handle long lines correctly: - - :::vim - set wrap - set textwidth=79 - set formatoptions=qrn1 - set colorcolumn=85 - -These lines manage my line wrapping settings and also show a colored column at -85 characters (so I can see when I write a too-long line of code). - -See [`:help fo-table`][fotable] and the Vimcasts on [soft wrapping][] and [hard wrapping][] for more information. - -[fotable]: http://vimdoc.sourceforge.net/htmldoc/change.html#fo-table -[soft wrapping]: http://vimcasts.org/episodes/soft-wrapping-text/ -[hard wrapping]: http://vimcasts.org/episodes/hard-wrapping-text/ - -Next comes something other TextMate refugees may like: - - :::vim - set list - set listchars=tab:▸\ ,eol:¬ - -This makes Vim show invisible characters with the same characters that TextMate -uses. You might need to adjust your color scheme so they're not too -distracting. [This Vimcast][vcinvis] has more information. - -[vcinvis]: http://vimcasts.org/episodes/show-invisibles/ - -New Vim users will want the following lines to teach them to do things right: - - :::vim - nnoremap - nnoremap - nnoremap - nnoremap - inoremap - inoremap - inoremap - inoremap - nnoremap j gj - nnoremap k gk - -This will disable the arrow keys while you're in normal mode to help you learn -to use `hjkl`. Trust me, you want to learn to use `hjkl`. Playing a lot of -[Nethack][] also helps. - -[Nethack]: http://www.nethack.org/ - -It also disables the arrow keys in insert mode to force you to get back into -normal mode the instant you're done inserting text, which is the "right way" to -do things. - -It also makes j and k work the way you expect instead of working in some -archaic "movement by file line instead of screen line" fashion. - -Next, get rid of that stupid goddamned help key that you will invaribly hit -*constantly* while aiming for escape: - - :::vim - inoremap - nnoremap - vnoremap - -I also like to make `;` do the same thing as `:` -- it's one less key to hit -every time I want to save a file: - - :::vim - nnoremap ; : - -I don't remap `:` back to `;` because it seems to break a bunch of plugins. - -Finally, I really like TextMate's "save on losing focus" feature. I can't -remember a time when I *didn't* want to save a file after tabbing away from my -editor (especially with version control and Vim's persistent undo): - - :::vim - au FocusLost * :wa - -Those are the most important bits of my `~/.vimrc` file. Next I'll talk about -the wonderful namespace of customization that is Vim's `` key. - -### Using the Leader - -Vim dedicates an entire keyboard key for user-specific customizations. This is -called the "leader" and by default it's mapped to `\`. As I mentioned in the -previous section I prefer to use `,` instead. - -Each person will find little things they type or execute often and want to -create shortcuts for those things. The leader is a kind of "namespace" to keep -those customizations separate and prevent them from shadowing default commands. - -Here are a few of the things I use leader commands for. You'll certainly have -different ideas than I do, but this might give you an idea of what you can do. - -I use `,W` to mean "strip all trailing whitespace in the current file" so I can -clean things up quickly: - - :::vim - nnoremap W :%s/\s\+$//:let @/='' - -I use Ack a lot (described below), so I mapped a leader key for it: - - :::vim - nnoremap a :Ack - -I work with HTML often, so I have `,ft` mapped to a "fold tag" function: - - :::vim - nnoremap ft Vatzf - -I also work with Nick Sergeant and he likes his CSS properties sorted, so -here's a `,S` mapping that sorts them for me: - - :::vim - nnoremap S ?{jV/^\s*\}?$k:sort:noh - -This next mapping imitates TextMates `Ctrl+Q` function to re-hardwrap -paragraphs of text: - - :::vim - nnoremap q gqip - -I have a `,v` mapping to reselect the text that was just pasted so I can -perform commands (like indentation) on it: - - :::vim - nnoremap v V`] - -This last mapping lets me quickly open up my `~/.vimrc` file in a vertically -split window so I can add new things to it on the fly. - - :::vim - nnoremap ev :e $MYVIMRC - -### Quicker Escaping - -One thing you'll find yourself constantly doing in Vim is moving from insert -mode to normal mode. The default way to do this is by hitting Escape, but that -key is out of the way and hard to hit. - -Another way is to use `Ctrl+C` or `Ctrl+[`, but I don't like using a chord for -something I press that often. - -I personally use `jj` to exit back to normal mode. The only time I've ever -actually tried to hit two `j`'s in a row is just now while writing this entry, -so it doesn't conflict with my normal typing at all: - - :::vim - inoremap jj - -### Working With Split Windows - -Being able to split my editor window and see more than one file at once is one -of the main reasons I switched to Vim. I know I'm not alone here after reading -various posts around the internet asking for this feature in TextMate. - -Vim's default commands for interacting with splits, however, are kind of -clunky. I've added a few mappings to my `~/.vimrc` that make things work a bit -smoother. - -This first mapping makes `,w` open a new vertical split and switch over to it. -Because really, how often do you split your window and not want to do something -in the new split? - - :::vim - nnoremap w vl - -You might notice that this mapping is for vertical splits. I almost *never* -use horizontal splits. All of my screens are widescreen, so I can fit several -files onscreen if they're split vertically. Horizontal splits don't let me see -enough of the file. If I *really* want a horizontal split I can use `s` -to get one. - -This next set of mappings maps `` to the commands needed to move -around your splits. If you remap your capslock key to `Ctrl` it makes for very -easy navigation. - - :::vim - nnoremap h - nnoremap j - nnoremap k - nnoremap l - -Aesthetics ----------- - -I know some people say: "looks don't matter, the only thing that's important is -functionality". That's completely wrong. - -If I'm looking at something for multiple hours each day it had damn well better -look pretty. There are three main steps to making Vim look pretty (for me, on -OS X -- your mileage may vary). - -First: get [MacVim][]. It makes Vim look (and behave) like a native Mac -application. It helps it avoid standing out like a sore thumb among your other -applications. Linux and Windows users will probably want gvim. - -Next: pick a decent font. I prefer 12 point [Menlo][]. You might have -a different preference, but at least try out a few and find one that has a nice -balance between line height (so you can get a good amount of lines on the -screen) and readability (slashed zeros, please). - -Finally: find a good color scheme. I've been using [a slightly modified -version][myMolokai] of [Molokai][] (a port of the TextMate [Monokai][] theme) -since I came back to Vim, but recently I've also started flirting with -[Mustang][] and [Clouds Midnight][]. Spend an hour and find something that -looks good to you. - -Here's what my current setup looks like: - -![Current Vim Setup Screenshot](/media/images{{ parent_url }}/vim.png "My Current Vim Setup") - -[Menlo]: http://arstechnica.com/apple/news/2009/06/font-changes-coming-to-mac-os-x-snow-leopard.ars -[myMolokai]: http://bitbucket.org/sjl/dotfiles/src/tip/vim/colors/molokai.vim -[Molokai]: http://www.vim.org/scripts/script.php?script_id=2340 -[Monokai]: http://www.monokai.nl/blog/2006/07/15/textmate-color-theme/ -[Mustang]: http://hcalves.deviantart.com/art/Mustang-Vim-Colorscheme-98974484 -[Clouds Midnight]: http://forr.st/~yZn - -Bundles I Use -------------- - -Vim has been around for a *long* time, and many people have written extensions -for it. Here are a few of the bundles I couldn't live without in my day-to-day -editing. - -### Pathogen - -First of all: [Tim Pope][] has created the wonderful [Pathogen][] plugin that -makes managing *other* Vim plugins painless. Instead of scattering their files -throughout your `~/.vim/` folder you can keep their files inside a single -folder in `~/.vim/bundles/`. - -Install it and keep your sanity. - -[Tim Pope]: http://github.com/tpope -[Pathogen]: http://github.com/tpope/vim-pathogen - -### PeepOpen - -One of TextMate's "killer features" is its `Cmd+T` key. It lets you open files -quickly by typing fragments of their names. - -There are a number of Vim plugins out there that try to emulate it. My -favorite is [PeepOpen][]. Yes, it's OS X only and costs money, but it's worth -it. It looks good and "just works". - -PeepOpen has some nifty little features, like showing Git metadata in the file -list. I've [offered a patch][] to add the same functionality for Mercurial but -haven't heard back from the developers. - -If you don't use OS X (or want a free alternative to PeepOpen) I hear the -[Command-T][] plugin is quite nice. - -[PeepOpen]: http://peepcode.com/products/peepopen -[offered a patch]: http://github.com/topfunky/PeepOpen-Issues/issues#issue/91 -[Command-T]: https://wincent.com/products/command-t - -### NERDTree - -One plugin you'll hear about in almost all of these "switching to Vim" blog -posts is [NERDTree][]. It's a little plugin for browsing files in your project, -and it works great. There's no better "TextMate file-drawer" plugin around. - -[NERDTree]: http://github.com/scrooloose/nerdtree - -### NERDCommenter - -Want to be able to comment and uncomment code with a few keypresses? You need -[NERDCommenter][]. It's surprising that Vim doesn't have decent commenting -functionality built in, but NERDCommenter fixes that. - -I really only use this plugin for one single function: "toggle comment" with -`c`. That function alone is worth installing it. - -[NERDCommenter]: http://github.com/scrooloose/nerdcommenter - -### Ack - -If you're a programmer and you don't know about [Ack][], you need to start -using it now. It's far, far better than grep. - -The [Ack plugin][ack.vim] for Vim integrates Ack with Vim's quickfix window so -you can easily search and jump to results. - -As I mentioned in the section about leader mappings I've got `,a` mapped to -bring up Ack all ready to search. - -[Ack]: http://betterthangrep.com/ -[ack.vim]: http://github.com/mileszs/ack.vim - -### Snipmate - -Another feature of TextMate that was *amazing* when I first saw it was -snippets. [SnipMate][] is a Vim plugin that emulates TextMate snippets. - -It also stores them in easily-version-controlled plain text files, which is -a bonus over TextMate's snippets. - -[SnipMate]: http://github.com/msanders/snipmate.vim - -### Sparkup - -[Sparkup][] is pretty much a great port of [Zen Coding][] for Vim. If you write -HTML at all this plugin will save you a ton of typing. - -In a nutshell it lets you type something like: - - :::text - div.content>h1.post-title+p{Sample Content} - -Press `Cmd+E` and it will expand to this: - - :::html -
-

-

Sample Content

-
- -It's far less typing and it adds up over time. - -[Sparkup]: http://github.com/rstacruz/sparkup -[Zen Coding]: http://code.google.com/p/zen-coding/ - -### Yankring - -Vim's copying and pasting functionality (which uses registers) is very, very -powerful, but it's not exactly "user friendly". The [YankRing][] plugin adds -a lot *more* power, but also adds a few features that make copying and pasting -much more pleasant. - -For example, after you paste some text you can replace that paste with the -previous item you copied with `Ctrl-P`. You can cycle back further by just -hitting `Ctrl-P` over and over. - -YankRing also shares your yanked text between Vim windows, which makes things -"just work" when you want to paste text from one window into another. - -You can also show a list of all your previously yanked text with `:YRShow`. -Mapping that command to a key is quite helpful: - - :::vim - nnoremap :YRShow - inoremap :YRShow - -YankRing offers a ton of other cool functionality but I haven't had the time or -motivation to really dig in and find out how to use it. - -[YankRing]: http://www.vim.org/scripts/script.php?script_id=1234 - -### Surround (and Repeat) - -Another of Time Pope's plugins that I use all the time is the [Surround][] -plugin. It adds another layer to Vim's physics: "surrounding items". - -For example, you can "change surrounding single quotes to double quotes" by -using `cs'"`. Or if you're writing something in Markdown and want to italicize -a word you can use `ysiW*`. - -All the nouns and verbs you already know can be used, which fits with Vim's -philosophy and makes it extremely powerful. - -The [Repeat][] plugin is necessary to make these surround actions repeatable -with `.`. - -[Surround]: http://github.com/tpope/vim-surround -[Repeat]: http://github.com/tpope/vim-repeat - -### Slime - -Emacs users often tout SLIME as one of Emacs' "killer features". When writing -LISP code it's unchallengably awesome. - -Vim doesn't have anything like SLIME. It's not integrated with any language the -way Emacs is coupled to LISP. You can, however, achieve *some* of the power of -SLIME in Vim. - -[Jonathan Palardy][] has written a little Vim plugin called [Slime.vim][] that -lets Vim easily communicate with a screen session, probably one running a REPL -for some language like LISP, Python or Ruby. - -The general idea is that you'll fire up LISP in a screen session, then go over -to Vim and use `Ctrl+C Ctrl+C` to put chunks of code in your Vim window into -the screen session. - -It works and is quite handy for when you want to experiment with the contents -of a file. Add a mapping for shoving over the entire file and you're all set to -bash out and test some code quickly. - -I personally use [a slightly modified version][mySlime] that removes a bit of -the flexibility I don't need to make it quicker to use. Maybe some day I'll -expand on it and push it up to BitBucket and GitHub in a repo of its own. - -[Slime.vim]: http://technotales.wordpress.com/2007/10/03/like-slime-for-vim/ -[Jonathan Palardy]: http://technotales.wordpress.com/ -[mySlime]: http://bitbucket.org/sjl/dotfiles/src/tip/vim/plugin/slime.vim - -### Scratch - -The [Scratch][] plugin adds a function to quickly open a "scratch" buffer that -will never be saved. - -I have it mapped to `` and use it with slime to quickly stick -a bit of code that I don't want in my actual file into the REPL. - -[Scratch]: http://www.vim.org/scripts/script.php?script_id=664 - -### Rainbow Parentheses - -[Rainbow Parentheses][] is a Vim plugin that colorizes parentheses, square -brackets, curly brackets and angle brackets according to their nesting. - -It's easiest to describe what this looks like with a screenshot: - -![Rainbow Parentheses Screenshot](/media/images{{ parent_url }}/rainbow.png "Rainbow Parentheses") - -I use [a slightly modified version][myRainbow]. I have it mapped to `R` -and off by default. When I'm dealing with a particularly hairy piece of code -that has lots of nesting I simply hit `R` and get some color that helps -me keep my place. - -[Rainbow Parentheses]: http://www.vim.org/scripts/script.php?script_id=1561 -[myRainbow]: http://bitbucket.org/sjl/dotfiles/src/tip/vim/bundle/rainbow/ - -Things I Want -------------- - -Although Vim is definitely my new favorite editor, this blog post wouldn't be -complete without a few complaints. - -If anyone fixes/implements any of these I'll gladly buy them several beers. - -### HTML Indentation that Doesn't Suck - -I write a lot of HTML. One thing that annoys the hell out of me is Vim's -"smart" indenting of HTML. I've tried every combination of `cindent`, -`smartindent` and `autoindent` and I can't seem to get Vim to behave sanely. - -The problem is that sometimes when I press return after a tag that I've already -created Vim will unindent the new line *and the previous line*. This is -excruciatingly annoying. - -All I want is the following: - -* When I press return, create a new line at the same indentation level as the - current one. Don't try to be clever and adjust the indent of new line in - any fashion. -* If I press tab at the beginning of a line, indent the *current line* by - one tabstop. -* If I press backspace at the beginning of a line, delete one tabstop on that - line only. - -Please, someone make this happen and restore my sanity. - -### Python Support for Slime - -Earlier I mentioned that I use the slime plugin. One language it doesn't work -very well with is Python, because it preserves whitespace when it moves the -code over the REPL. - -This is a problem with Python because if you try to move over an indented block -of code you'll get "unexpected indent" problems. - -I'd love for someone to tweak slime so it intelligently unindents Python code. -I have some ideas on how this could work but unfortunately I don't have the -time to implement them. - -### Gundo - -A little-known fact about Vim is that it doesn't keep a list of your undo -history, it keeps a *tree*. - -If I make 5 changes, undo 2 and then make 2 more Vim keeps track of *all* of -them. You can use `:undolist` to see the list of leaves in this "undo tree". - -What I want is for someone to make an awesome "graphical undo" plugin that -shows a graph of the undo tree, much like Mercurial's graphlog extension shows -a graph of the changesets in a repository. - -This graph would actually be simpler than Mercurial's graphlog because there -are no merges to deal with. - -For bonus points let me browse the tree with `j` and `k`, press `p` to preview -a diff of what would happen if I went back to that version, and press `` to -actually go back. - -**UPDATE:** I got tired of waiting, so I [wrote it myself][Gundo]. - -[Gundo]: http://bitbucket.org/sjl/gundo.vim/ - -### A Mercurial Version of Fugitive - -The final thing I'd like is for someone to make a Mercurial plugin for Vim that -is as awesome as Tim Pope's [Fugitive][]. - -There's some Mercurial support for Vim out there, but for the most part it's -tied up in plugins that support multiple version control systems. - -Any plugin that tries to support every system will fail at supporting one -single system perfectly, so I'd love for someone to come along and make an -*awesome* Vim plugin that lets you use Mercurial to its fullest while -inside Vim. - -I simply don't know vimscript well enough to do this on my own, but if someone -else is interested I can certainly help out on the Mercurial and user interface -aspects. - -[Fugitive]: http://github.com/tpope/vim-fugitive - -Overall Thoughts ----------------- - -Transitioning back to Vim after using TextMate for a few years wasn't trivial, -but I feel that it was worth the effort. I have features like split windows -now and there are plugins that can provide anything I miss from TextMate. - -If TextMate 2 ever gets released it would have to do some pretty amazing things -for me to want to switch back. - -Overall I'm very happy with Vim. There are a lot of things that could be -improved, especially with the default settings (who cares about vi -compatibility in 2010?), but a text editor is worth a learning curve. - -In my eyes Vim has two huge advantages over almost any other editor out there. -The first is it's *huge* ecosystem of plugins and syntax definitions, which is -only rivaled by Emacs (and *possibly* TextMate, though I doubt it). The second -is it's "modal editing" philosophy, which is extremely powerful and hasn't been -adopted by any other mainstream editor (except for *very* recent versions of -E). - -If you've got questions or comments you should [find me on Twitter][twsl] and -let me know. - -[twsl]: {{links.twsl}} - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2010/09/coming-home-to-vim.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2010/09/coming-home-to-vim.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,964 @@ ++++ +title = "Coming Home to Vim" +snip = "I'm sorry I ever left, baby." +date = 2010-09-20T18:15:00Z +draft = false + ++++ + +I'm a programmer. I work with text files for 6-12 hours every weekday so I care +about the text editor I use. If switching to a different editor can increase my +efficiency by even 10% it would save a good chunk of my time and let me get +back to making cool things. + +I don't buy the "you're thinking 90% of the time and only typing 10% of the +time, so your editor doesn't really matter" argument. Even if the premise is +true, the conclusion is wrong. + +If I think for 10 minutes and then start typing, I want the typing to take the +shortest time possible so I can get back to thinking. Any time I spend typing +is an interruption that I want to minimize so I can keep my train of thought. + +I recently started using [Vim][] as my primary editor. As I'm adjusting I'm +finding a lot of [the][arrows] [blog][nutheads] [posts][mind] [people][katz] +[have][nvie] [written][grok] [about][home] Vim very helpful, so I'm hoping this +post will help people too. + +[Vim]: http://www.vim.org/ +[arrows]: http://jeetworks.org/node/89 +[nutheads]: http://www.viemu.com/a-why-vi-vim.html +[mind]: http://wekeroad.com/2010/07/29/vim-is-your-daddy/ +[katz]: http://yehudakatz.com/2010/07/29/everyone-who-tried-to-convince-me-to-use-vim-was-wrong/ +[nvie]: http://nvie.com/posts/how-i-boosted-my-vim/ +[grok]: http://stackoverflow.com/questions/1218390/what-is-your-most-productive-shortcut-with-vim/1220118#1220118 +[home]: http://weblog.jamisbuck.org/2008/10/10/coming-home-to-vim + +{{% toc %}} + +Some Background About Me +------------------------ + +I used Vim exclusively from about 2002 to 2005. I never really learned much +about it and mostly used it as a glorified [nano][]. + +In 2006 I decided it was time for a change. I wanted an editor that "fit" with +OS X. I tried [SubEthaEdit][] for a while and it was kind of cool. The +collaborative editing was more polished than any other version I've seen, +though I didn't have much use for it. + +As cool as SubEthaEdit was, I always felt like there was a better editor out +there for me. Then I found [TextMate][]. + +At the time I thought TextMate was amazing (and it really was). It *looked* +like a Mac application. It was super easy to learn (all the OS X text movement +commands just worked). There were bundles for everything. + +I still used Vim occasionally (like when I had a job and only had a Windows +machine), but for the most part I was a TextMate kid. + +After using TextMate for four years I decided it was time for another change. +Development on TextMate has stalled while TextMate Forever (err, TextMate 2) is +in the works. I started thinking about Vim (and reading blog posts by others +about how they switched back) and decided to give it another try. + +A few months ago I bought [Learning the vi and Vim Editors][] and went +cold-turkey with Vim as my main text editor. I haven't looked back since. + +[nano]: http://en.wikipedia.org/wiki/Nano_(text_editor) +[SubEthaEdit]: http://www.codingmonkeys.de/subethaedit/ +[TextMate]: http://macromates.com/ +[Learning the vi and Vim Editors]: http://amzn.com/059652983X + +Why I Switched to TextMate +-------------------------- + +As I mentioned before, there were a few reasons I switched to TextMate in the +first place. + +First of all: it looks and acts like a real OS X application. You can drag +files onto the icon to open them, it supports all the default OS X text +movement commands, and the chrome looks like it belongs on OS X. I don't know +if [MacVim][] was around at the time, but if it was I never found it. + +This matters. The effort required to get started with a new editor is +dramatically reduced if that editor supports all the conventions of the +operating system you use every day. + +It also has a vibrant community of people writing bundles for it, so there was +support for almost anything I wanted to use. + +[MacVim]: http://code.google.com/p/macvim/ + +Why I Came Back to Vim +---------------------- + +I came back to Vim for a number of reasons. + +First of all: I started reading about and desiring features that didn't seem to +be coming to TextMate any time soon. The biggest of those was split windows. +I saw the appeal of split windows right away and now that I use them I can't +imagine not having them. + +Another reason is version control. A while ago I started keeping my +[dotfiles][] in a [Mercurial][] repository which makes it extremely easy to set +up new machines. Vim keeps all of its configuration in two simple places: +a simple `~/.vimrc` file and a `~/.vim` directory full of plain text files. + +Vim's plain text configuration files and plugins are very easy to version +control. They diff well, unlike TextMate's multiple `Bundles` directories and +ugly XML file format. + +Vim's community has been around far longer than TextMate's so there's an even +richer set of plugins, bundles and syntax files available. + +Another reason people give for enjoying Vim is that it runs everywhere. Yes, +it's nice to have my favorite editor available when I SSH into a machine, but +it's not a huge deal if I don't. I almost always edit files on my own machine +and then deploy them with an automated script of some kind, so editing on +a server is very rare for me. + +[dotfiles]: http://bitbucket.org/sjl/dotfiles/ +[Mercurial]: http://hg-scm.org/ + +Core Differences +---------------- + +For me, Vim and TextMate are very different. TextMate relies mainly on +keyboard shortcuts (involving Shift, Ctrl, Cmd and Alt) to do more than simply +edit text. Vim uses a "modal editing" idea to accomplish the same task. + +TextMate's philosophy is much the same as any normal OS X application's, and +I said that acting like the OS reduces the barrier of entry for a text editor, +so why do I prefer Vim's philosophy? + +First I should prefix my answer with this: I'm not averse to a learning curve +for something as important to me as a text editor. Yes, it's awesome if +I don't need to unnecessarily rewire my brain, but if it's going to save me +a lot of time then I'm willing to cut a bit of slack here. + +My problem with TextMate's philosophy can be summed up in one word: +"shadowing". + +If I want to define a new command in a bundle (or if someone else has already +done it), I'm never really sure if I'm "overwriting" some other command that my +fingers know but my brain doesn't really register. + +By the time I try to use the default command it could be days or weeks after +I defined the custom one, and I need to figure out what I did all over again. +It's quite frustrating. + +Vim's "insert" mode means that when I'm editing text I'm using the normal OS +X text movement shortcuts that I know and love. When I want to do something +special I enter normal mode and don't have to worry about shadowed commands +(Vim's leader key also helps with this). + +Vim's "Feeling" +--------------- + +Vim's normal mode has a unique "feeling" that I haven't seen in any other text +editor (although [E][] may [change that][modalE]). I've heard it described in +a number of ways, and I think that while no one analogy fully explains it, +together they give a pretty good description. + +[E]: http://www.e-texteditor.com/ +[modalE]: http://e-texteditor.com/blog/2010/beyond-vi + +### A "Language" of Text Editing + +One way to think about Vim's normal-mode commands is like a language. You have +"verbs" and "nouns". For example: the "change" command (`c`) would be a verb +and the "word" item (`w`) is a noun. You can combine them to form imperative +sentences that talk about what you want to do with your text. + +The wonderful part about this is that whenever you learn a new verb (like +"delete" (`d`)) you can immediately apply it to all the nouns you know, and +vice versa. + +Vim also has "adjectives" like "inside" and "around" (`i` and `a`) that let you +craft sentences like "change inside parenthesis" (`ci(` or `cib`). Once you +learn one of these you can immediately apply it to all the verbs and nouns you +already know. + +### The Physics of Text + +The second way to describe the feeling of Vim is "physics". It's a much less +concrete description but I think it's still useful. + +If I throw a bowling ball into the air it behaves much the same way as if +I throw a turkey sandwich. They might have very different effects when they +land on someone but the act of throwing them is pretty much the same. + +Vim's operators/actions/verbs act the same way. `daw` has a very different +effect than `da{` but they can both be understood with a basic underlying +principle: `da` will "delete around ``". + +### "Programming" Your Editing + +The last way that I hear people talking about Vim's editing is: "it feels like +you're 'programming' your text". + +This analogy is often made by programmers (obviously) and I think it works on +more than one level. + +First: Vim's basic editing commands can be likened to function calls. `daw` +can be thought of a running a function like `delete(type='word', around=True)`. +I don't think this way (I prefer the language and physics analogies) but other +people do. + +Although I don't think of the *basic* Vim commands as programming I do see +some aspects of it in other areas. For example: say I define a leader command +in my `~/.vimrc` file: + +```vim +nnoremap 1 yypVr= +``` + +I think of this as defining `1` to be a function that performs the +following actions: + +* Yank/copy the current line. +* Paste it below (and move down to the pasted version). +* Select the copied line. +* Replace every character with `=`. + +Another more obvious "programming" aspect of Vim is creating macros. Macros are +like tiny little functions you create to help you when you're editing a file. +You can define them, execute them, and even edit them by pasting, editing and +yanking them back. + +Getting Started +--------------- + +When I switched back to Vim I did a few things that helped make my transition +stick. + +First: I bought [Learning the vi and Vim Editors][] and read it cover to cover. +It helps quite a bit to know the history behind Vim, and the book also teaches +you the basics of Vim's editing commands. + +I also looked through [the Vim subreddit][rvim] to find some good blog posts about +switching. A lot of people have written about it and it was very helpful to +read about their experiences. + +And finally, the most important part: I switched cold-turkey. Vim runs everywhere, so +there's really no excuse for not doing this. Every decent OS has a version of +Vim that makes insert mode "just work" for their OS (like gvim or MacVim), so +the barrier to entry these days is pretty low. + +You can get Vim and start using it right away by staying in insert mode most of +the time. Then you can start learning its real features at your own pace. +Trying to use two different editors at once just slows down the learning +process and makes you less productive in both. + +[rvim]: http://reddit.com/r/vim/ + +Making Vim More Useful +---------------------- + +One of the flaws of Vim is that it takes quite a bit of configuration to make +it behave in a decent way. The defaults are backwards-compatible with vi (i.e. +older than most college students) and not very helpful. + +Here are a few of the things I do to make Vim a bit more palatable. + +**A quick side note:** even if you don't use Vim, you *need* to [remap your +capslock key][capslock] to something useful. Just do it, you'll thank me. + +[capslock]: http://c2.com/cgi/wiki?RemapCapsLock + +### Important .vimrc Lines + +I won't go through every line in [my `~/.vimrc` file][sjlvimrc], but here are some of the +lines that I simply could not live without. + +[sjlvimrc]: http://bitbucket.org/sjl/dotfiles/src/tip/vim/.vimrc + +First, a few lines that you absolutely must have: + +```vim +filetype off +call pathogen#runtime_append_all_bundles() +filetype plugin indent on + +set nocompatible + +set modelines=0 +``` + +The `filetype` and `call` lines are for loading Pathogen, which is described in +the bundles section. See [Pathogen's docs][pathogendocs] to learn about why the +first `filetype` line is there. + +[pathogendocs]: http://www.vim.org/scripts/script.php?script_id=2332 + +`set nocompatible` gets rid of all the crap that Vim does to be vi compatible. +It's 2010 — we don't need to be compatible with vi at the expense of +functionality any more. + +The `modelines` bit prevents some [security exploits][] having to do with +modelines in files. I never use modelines so I don't miss any functionality +here. + +[security exploits]: http://lists.alioth.debian.org/pipermail/pkg-vim-maintainers/2007-June/004020.html + +Next I set my tab settings: + +```vim +set tabstop=4 +set shiftwidth=4 +set softtabstop=4 +set expandtab +``` + +I like all tabs to expand to four spaces. Check out [this Vimcast][vctabs] to +learn more about each of these options. + +[vctabs]: http://vimcasts.org/episodes/tabs-and-spaces/ + +Next are a few options that just make things better: + +```vim +set encoding=utf-8 +set scrolloff=3 +set autoindent +set showmode +set showcmd +set hidden +set wildmenu +set wildmode=list:longest +set visualbell +set cursorline +set ttyfast +set ruler +set backspace=indent,eol,start +set laststatus=2 +set relativenumber +set undofile +``` + +Each of these lines are basically to make Vim behave in a sane manner. The two +"interesting" ones are the last two, and both deal with features that are +[new in Vim 7.3][vim73]. + +[vim73]: http://groups.google.com/group/vim_announce/browse_thread/thread/66c02efd1523554b + +`relativenumber` changes Vim's line number column to display how far away each +line is from the current one, instead of showing the absolute line number. + +I almost never care what numeric line I'm on in a file (and if I do I can see +it in the status line), so I don't miss the normal line numbers. I *do* care +how far away a particular line might be, because it tells me what number I need +to use with motion commands like `dd`. + +`undofile` tells Vim to create `.un~` files whenever you edit a file. +These files contain undo information so you can undo previous actions even +after you close and reopen a file. + +Next I change the `` key: + +```vim +let mapleader = "," +``` + +For me `,` is easier to type than `\`. I use leader commands constantly so it's +worth changing. + +The next thing I do is tame searching/moving: + +```vim +nnoremap / /\v +vnoremap / /\v +set ignorecase +set smartcase +set gdefault +set incsearch +set showmatch +set hlsearch +nnoremap :noh +nnoremap % +vnoremap % +``` + +The first two lines fix Vim's horribly broken default regex "handling" by +automatically inserting a `\v` before any string you search for. This turns +off Vim's crazy default regex characters and makes searches use normal regexes. +I already know Perl/Python compatible regex formatting, why would I want to +learn another scheme? + +`ignorecase` and `smartcase` together make Vim deal with case-sensitive search +intelligently. If you search for an all-lowercase string your search will be +case-insensitive, but if one or more characters is uppercase the search will be +case-sensitive. Most of the time this does what you want. + +`gdefault` applies substitutions globally on lines. For example, instead of +`:%s/foo/bar/g` you just type `:%s/foo/bar/`. This is almost always what you +want (when was the last time you wanted to only replace the first occurrence of +a word on a line?) and if you need the previous behavior you just tack on the +`g` again. + +`incsearch`, `showmatch` and `hlsearch` work together to highlight search +results (as you type). It's really quite handy, as long as you have the next +line as well. + +The `` mapping makes it easy to clear out a search by typing +`,`. This gets rid of the distracting highlighting once I've found what +I'm looking for. + +The last two lines make the tab key match bracket pairs. I use this to move +around all the time and `` is a hell of a lot easier to type than `%`. + +The next section makes Vim handle long lines correctly: + +```vim +set wrap +set textwidth=79 +set formatoptions=qrn1 +set colorcolumn=85 +``` + +These lines manage my line wrapping settings and also show a colored column at +85 characters (so I can see when I write a too-long line of code). + +See [`:help fo-table`][fotable] and the Vimcasts on [soft wrapping][] and [hard wrapping][] for more information. + +[fotable]: http://vimdoc.sourceforge.net/htmldoc/change.html#fo-table +[soft wrapping]: http://vimcasts.org/episodes/soft-wrapping-text/ +[hard wrapping]: http://vimcasts.org/episodes/hard-wrapping-text/ + +Next comes something other TextMate refugees may like: + +```vim +set list +set listchars=tab:▸\ ,eol:¬ +``` + +This makes Vim show invisible characters with the same characters that TextMate +uses. You might need to adjust your color scheme so they're not too +distracting. [This Vimcast][vcinvis] has more information. + +[vcinvis]: http://vimcasts.org/episodes/show-invisibles/ + +New Vim users will want the following lines to teach them to do things right: + +```vim +nnoremap +nnoremap +nnoremap +nnoremap +inoremap +inoremap +inoremap +inoremap +nnoremap j gj +nnoremap k gk +``` + +This will disable the arrow keys while you're in normal mode to help you learn +to use `hjkl`. Trust me, you want to learn to use `hjkl`. Playing a lot of +[Nethack][] also helps. + +[Nethack]: http://www.nethack.org/ + +It also disables the arrow keys in insert mode to force you to get back into +normal mode the instant you're done inserting text, which is the "right way" to +do things. + +It also makes j and k work the way you expect instead of working in some +archaic "movement by file line instead of screen line" fashion. + +Next, get rid of that stupid goddamned help key that you will invaribly hit +*constantly* while aiming for escape: + +```vim +inoremap +nnoremap +vnoremap +``` + +I also like to make `;` do the same thing as `:` — it's one less key to hit +every time I want to save a file: + +```vim +nnoremap ; : +``` + +I don't remap `:` back to `;` because it seems to break a bunch of plugins. + +Finally, I really like TextMate's "save on losing focus" feature. I can't +remember a time when I *didn't* want to save a file after tabbing away from my +editor (especially with version control and Vim's persistent undo): + +```vim +au FocusLost * :wa +``` + +Those are the most important bits of my `~/.vimrc` file. Next I'll talk about +the wonderful namespace of customization that is Vim's `` key. + +### Using the Leader + +Vim dedicates an entire keyboard key for user-specific customizations. This is +called the "leader" and by default it's mapped to `\`. As I mentioned in the +previous section I prefer to use `,` instead. + +Each person will find little things they type or execute often and want to +create shortcuts for those things. The leader is a kind of "namespace" to keep +those customizations separate and prevent them from shadowing default commands. + +Here are a few of the things I use leader commands for. You'll certainly have +different ideas than I do, but this might give you an idea of what you can do. + +I use `,W` to mean "strip all trailing whitespace in the current file" so I can +clean things up quickly: + +```vim +nnoremap W :%s/\s\+$//:let @/='' +``` + +I use Ack a lot (described below), so I mapped a leader key for it: + +```vim +nnoremap a :Ack +``` + +I work with HTML often, so I have `,ft` mapped to a "fold tag" function: + +```vim +nnoremap ft Vatzf +``` + +I also work with Nick Sergeant and he likes his CSS properties sorted, so +here's a `,S` mapping that sorts them for me: + +```vim +nnoremap S ?{jV/^\s*\}?$k:sort:noh +``` + +This next mapping imitates TextMates `Ctrl+Q` function to re-hardwrap +paragraphs of text: + +```vim +nnoremap q gqip +``` + +I have a `,v` mapping to reselect the text that was just pasted so I can +perform commands (like indentation) on it: + +```vim +nnoremap v V`] +``` + +This last mapping lets me quickly open up my `~/.vimrc` file in a vertically +split window so I can add new things to it on the fly. + +```vim +nnoremap ev :e $MYVIMRC +``` + +### Quicker Escaping + +One thing you'll find yourself constantly doing in Vim is moving from insert +mode to normal mode. The default way to do this is by hitting Escape, but that +key is out of the way and hard to hit. + +Another way is to use `Ctrl+C` or `Ctrl+[`, but I don't like using a chord for +something I press that often. + +I personally use `jj` to exit back to normal mode. The only time I've ever +actually tried to hit two `j`'s in a row is just now while writing this entry, +so it doesn't conflict with my normal typing at all: + +```vim +inoremap jj +``` + +### Working With Split Windows + +Being able to split my editor window and see more than one file at once is one +of the main reasons I switched to Vim. I know I'm not alone here after reading +various posts around the internet asking for this feature in TextMate. + +Vim's default commands for interacting with splits, however, are kind of +clunky. I've added a few mappings to my `~/.vimrc` that make things work a bit +smoother. + +This first mapping makes `,w` open a new vertical split and switch over to it. +Because really, how often do you split your window and not want to do something +in the new split? + +```vim +nnoremap w vl +``` + +You might notice that this mapping is for vertical splits. I almost *never* +use horizontal splits. All of my screens are widescreen, so I can fit several +files onscreen if they're split vertically. Horizontal splits don't let me see +enough of the file. If I *really* want a horizontal split I can use `s` +to get one. + +This next set of mappings maps `` to the commands needed to move +around your splits. If you remap your capslock key to `Ctrl` it makes for very +easy navigation. + +```vim +nnoremap h +nnoremap j +nnoremap k +nnoremap l +``` + +Aesthetics +---------- + +I know some people say: "looks don't matter, the only thing that's important is +functionality". That's completely wrong. + +If I'm looking at something for multiple hours each day it had damn well better +look pretty. There are three main steps to making Vim look pretty (for me, on +OS X — your mileage may vary). + +First: get [MacVim][]. It makes Vim look (and behave) like a native Mac +application. It helps it avoid standing out like a sore thumb among your other +applications. Linux and Windows users will probably want gvim. + +Next: pick a decent font. I prefer 12 point [Menlo][]. You might have +a different preference, but at least try out a few and find one that has a nice +balance between line height (so you can get a good amount of lines on the +screen) and readability (slashed zeros, please). + +Finally: find a good color scheme. I've been using [a slightly modified +version][myMolokai] of [Molokai][] (a port of the TextMate [Monokai][] theme) +since I came back to Vim, but recently I've also started flirting with +[Mustang][] and [Clouds Midnight][]. Spend an hour and find something that +looks good to you. + +Here's what my current setup looks like: + +![Current Vim Setup Screenshot](/media/images/blog/2010/09/vim.png "My Current Vim Setup") + +[Menlo]: http://arstechnica.com/apple/news/2009/06/font-changes-coming-to-mac-os-x-snow-leopard.ars +[myMolokai]: http://bitbucket.org/sjl/dotfiles/src/tip/vim/colors/molokai.vim +[Molokai]: http://www.vim.org/scripts/script.php?script_id=2340 +[Monokai]: http://www.monokai.nl/blog/2006/07/15/textmate-color-theme/ +[Mustang]: http://hcalves.deviantart.com/art/Mustang-Vim-Colorscheme-98974484 +[Clouds Midnight]: http://forr.st/~yZn + +Bundles I Use +------------- + +Vim has been around for a *long* time, and many people have written extensions +for it. Here are a few of the bundles I couldn't live without in my day-to-day +editing. + +### Pathogen + +First of all: [Tim Pope][] has created the wonderful [Pathogen][] plugin that +makes managing *other* Vim plugins painless. Instead of scattering their files +throughout your `~/.vim/` folder you can keep their files inside a single +folder in `~/.vim/bundles/`. + +Install it and keep your sanity. + +[Tim Pope]: http://github.com/tpope +[Pathogen]: http://github.com/tpope/vim-pathogen + +### PeepOpen + +One of TextMate's "killer features" is its `Cmd+T` key. It lets you open files +quickly by typing fragments of their names. + +There are a number of Vim plugins out there that try to emulate it. My +favorite is [PeepOpen][]. Yes, it's OS X only and costs money, but it's worth +it. It looks good and "just works". + +PeepOpen has some nifty little features, like showing Git metadata in the file +list. I've [offered a patch][] to add the same functionality for Mercurial but +haven't heard back from the developers. + +If you don't use OS X (or want a free alternative to PeepOpen) I hear the +[Command-T][] plugin is quite nice. + +[PeepOpen]: http://peepcode.com/products/peepopen +[offered a patch]: http://github.com/topfunky/PeepOpen-Issues/issues#issue/91 +[Command-T]: https://wincent.com/products/command-t + +### NERDTree + +One plugin you'll hear about in almost all of these "switching to Vim" blog +posts is [NERDTree][]. It's a little plugin for browsing files in your project, +and it works great. There's no better "TextMate file-drawer" plugin around. + +[NERDTree]: http://github.com/scrooloose/nerdtree + +### NERDCommenter + +Want to be able to comment and uncomment code with a few keypresses? You need +[NERDCommenter][]. It's surprising that Vim doesn't have decent commenting +functionality built in, but NERDCommenter fixes that. + +I really only use this plugin for one single function: "toggle comment" with +`c`. That function alone is worth installing it. + +[NERDCommenter]: http://github.com/scrooloose/nerdcommenter + +### Ack + +If you're a programmer and you don't know about [Ack][], you need to start +using it now. It's far, far better than grep. + +The [Ack plugin][ack.vim] for Vim integrates Ack with Vim's quickfix window so +you can easily search and jump to results. + +As I mentioned in the section about leader mappings I've got `,a` mapped to +bring up Ack all ready to search. + +[Ack]: http://betterthangrep.com/ +[ack.vim]: http://github.com/mileszs/ack.vim + +### Snipmate + +Another feature of TextMate that was *amazing* when I first saw it was +snippets. [SnipMate][] is a Vim plugin that emulates TextMate snippets. + +It also stores them in easily-version-controlled plain text files, which is +a bonus over TextMate's snippets. + +[SnipMate]: http://github.com/msanders/snipmate.vim + +### Sparkup + +[Sparkup][] is pretty much a great port of [Zen Coding][] for Vim. If you write +HTML at all this plugin will save you a ton of typing. + +In a nutshell it lets you type something like: + +```text +div.content>h1.post-title+p{Sample Content} +``` + +Press `Cmd+E` and it will expand to this: + +```html +
+

+

Sample Content

+
+``` + +It's far less typing and it adds up over time. + +[Sparkup]: http://github.com/rstacruz/sparkup +[Zen Coding]: http://code.google.com/p/zen-coding/ + +### Yankring + +Vim's copying and pasting functionality (which uses registers) is very, very +powerful, but it's not exactly "user friendly". The [YankRing][] plugin adds +a lot *more* power, but also adds a few features that make copying and pasting +much more pleasant. + +For example, after you paste some text you can replace that paste with the +previous item you copied with `Ctrl-P`. You can cycle back further by just +hitting `Ctrl-P` over and over. + +YankRing also shares your yanked text between Vim windows, which makes things +"just work" when you want to paste text from one window into another. + +You can also show a list of all your previously yanked text with `:YRShow`. +Mapping that command to a key is quite helpful: + +```vim +nnoremap :YRShow +inoremap :YRShow +``` + +YankRing offers a ton of other cool functionality but I haven't had the time or +motivation to really dig in and find out how to use it. + +[YankRing]: http://www.vim.org/scripts/script.php?script_id=1234 + +### Surround (and Repeat) + +Another of Time Pope's plugins that I use all the time is the [Surround][] +plugin. It adds another layer to Vim's physics: "surrounding items". + +For example, you can "change surrounding single quotes to double quotes" by +using `cs'"`. Or if you're writing something in Markdown and want to italicize +a word you can use `ysiW*`. + +All the nouns and verbs you already know can be used, which fits with Vim's +philosophy and makes it extremely powerful. + +The [Repeat][] plugin is necessary to make these surround actions repeatable +with `.`. + +[Surround]: http://github.com/tpope/vim-surround +[Repeat]: http://github.com/tpope/vim-repeat + +### Slime + +Emacs users often tout SLIME as one of Emacs' "killer features". When writing +LISP code it's unchallengably awesome. + +Vim doesn't have anything like SLIME. It's not integrated with any language the +way Emacs is coupled to LISP. You can, however, achieve *some* of the power of +SLIME in Vim. + +[Jonathan Palardy][] has written a little Vim plugin called [Slime.vim][] that +lets Vim easily communicate with a screen session, probably one running a REPL +for some language like LISP, Python or Ruby. + +The general idea is that you'll fire up LISP in a screen session, then go over +to Vim and use `Ctrl+C Ctrl+C` to put chunks of code in your Vim window into +the screen session. + +It works and is quite handy for when you want to experiment with the contents +of a file. Add a mapping for shoving over the entire file and you're all set to +bash out and test some code quickly. + +I personally use [a slightly modified version][mySlime] that removes a bit of +the flexibility I don't need to make it quicker to use. Maybe some day I'll +expand on it and push it up to BitBucket and GitHub in a repo of its own. + +[Slime.vim]: http://technotales.wordpress.com/2007/10/03/like-slime-for-vim/ +[Jonathan Palardy]: http://technotales.wordpress.com/ +[mySlime]: http://bitbucket.org/sjl/dotfiles/src/tip/vim/plugin/slime.vim + +### Scratch + +The [Scratch][] plugin adds a function to quickly open a "scratch" buffer that +will never be saved. + +I have it mapped to `` and use it with slime to quickly stick +a bit of code that I don't want in my actual file into the REPL. + +[Scratch]: http://www.vim.org/scripts/script.php?script_id=664 + +### Rainbow Parentheses + +[Rainbow Parentheses][] is a Vim plugin that colorizes parentheses, square +brackets, curly brackets and angle brackets according to their nesting. + +It's easiest to describe what this looks like with a screenshot: + +![Rainbow Parentheses Screenshot](/media/images/blog/2010/09/rainbow.png "Rainbow Parentheses") + +I use [a slightly modified version][myRainbow]. I have it mapped to `R` +and off by default. When I'm dealing with a particularly hairy piece of code +that has lots of nesting I simply hit `R` and get some color that helps +me keep my place. + +[Rainbow Parentheses]: http://www.vim.org/scripts/script.php?script_id=1561 +[myRainbow]: http://bitbucket.org/sjl/dotfiles/src/tip/vim/bundle/rainbow/ + +Things I Want +------------- + +Although Vim is definitely my new favorite editor, this blog post wouldn't be +complete without a few complaints. + +If anyone fixes/implements any of these I'll gladly buy them several beers. + +### HTML Indentation that Doesn't Suck + +I write a lot of HTML. One thing that annoys the hell out of me is Vim's +"smart" indenting of HTML. I've tried every combination of `cindent`, +`smartindent` and `autoindent` and I can't seem to get Vim to behave sanely. + +The problem is that sometimes when I press return after a tag that I've already +created Vim will unindent the new line *and the previous line*. This is +excruciatingly annoying. + +All I want is the following: + +* When I press return, create a new line at the same indentation level as the + current one. Don't try to be clever and adjust the indent of new line in + any fashion. +* If I press tab at the beginning of a line, indent the *current line* by + one tabstop. +* If I press backspace at the beginning of a line, delete one tabstop on that + line only. + +Please, someone make this happen and restore my sanity. + +### Python Support for Slime + +Earlier I mentioned that I use the slime plugin. One language it doesn't work +very well with is Python, because it preserves whitespace when it moves the +code over the REPL. + +This is a problem with Python because if you try to move over an indented block +of code you'll get "unexpected indent" problems. + +I'd love for someone to tweak slime so it intelligently unindents Python code. +I have some ideas on how this could work but unfortunately I don't have the +time to implement them. + +### Gundo + +A little-known fact about Vim is that it doesn't keep a list of your undo +history, it keeps a *tree*. + +If I make 5 changes, undo 2 and then make 2 more Vim keeps track of *all* of +them. You can use `:undolist` to see the list of leaves in this "undo tree". + +What I want is for someone to make an awesome "graphical undo" plugin that +shows a graph of the undo tree, much like Mercurial's graphlog extension shows +a graph of the changesets in a repository. + +This graph would actually be simpler than Mercurial's graphlog because there +are no merges to deal with. + +For bonus points let me browse the tree with `j` and `k`, press `p` to preview +a diff of what would happen if I went back to that version, and press `` to +actually go back. + +**UPDATE:** I got tired of waiting, so I [wrote it myself][Gundo]. + +[Gundo]: http://bitbucket.org/sjl/gundo.vim/ + +### A Mercurial Version of Fugitive + +The final thing I'd like is for someone to make a Mercurial plugin for Vim that +is as awesome as Tim Pope's [Fugitive][]. + +There's some Mercurial support for Vim out there, but for the most part it's +tied up in plugins that support multiple version control systems. + +Any plugin that tries to support every system will fail at supporting one +single system perfectly, so I'd love for someone to come along and make an +*awesome* Vim plugin that lets you use Mercurial to its fullest while +inside Vim. + +I simply don't know vimscript well enough to do this on my own, but if someone +else is interested I can certainly help out on the Mercurial and user interface +aspects. + +[Fugitive]: http://github.com/tpope/vim-fugitive + +Overall Thoughts +---------------- + +Transitioning back to Vim after using TextMate for a few years wasn't trivial, +but I feel that it was worth the effort. I have features like split windows +now and there are plugins that can provide anything I miss from TextMate. + +If TextMate 2 ever gets released it would have to do some pretty amazing things +for me to want to switch back. + +Overall I'm very happy with Vim. There are a lot of things that could be +improved, especially with the default settings (who cares about vi +compatibility in 2010?), but a text editor is worth a learning curve. + +In my eyes Vim has two huge advantages over almost any other editor out there. +The first is it's *huge* ecosystem of plugins and syntax definitions, which is +only rivaled by Emacs (and *possibly* TextMate, though I doubt it). The second +is it's "modal editing" philosophy, which is extremely powerful and hasn't been +adopted by any other mainstream editor (except for *very* recent versions of +E). + +If you've got questions or comments you should [find me on Twitter][twsl] and +let me know. + +[twsl]: {{links.twsl}} + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2010/09/making-my-site-sing.html --- a/content/blog/2010/09/making-my-site-sing.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,229 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "Making My Site Sing" - snip: "Designing with music." - created: 2010-09-08 20:10:00 -%} - -{% block article %} - -Every year or so I get the urge to redesign my site. It's only been seven -months since the last time, so I guess working with [Nick Sergeant][] and -[Ali Ali][] at [Dumbwaiter Design][] has made me less satisfied than usual -with my previous design. - -I didn't switch frameworks this time -- I'm still [using Hyde][] to generate -a set of flat HTML files. I only changed the structure and layout of the -rendered site. - -I'm a programmer, not a designer, but I think it looks fairly good. I wanted -to write about how I went about designing the site because it might help other -programmers. - -[Nick Sergeant]: http://nick.sg/ -[Ali Ali]: http://alialithinks.com/ -[Dumbwaiter Design]: http://dwaiter.com/ -[using Hyde]: /blog/2010/01/moving-from-django-to-hyde/ - -[TOC] - -Finding a Starting Point ------------------------- - -The first thing I had to do was come up with a basic idea for the style and -typography of the site. - -I decided to go with my usual style: dark text on a light background, very few -images, and an overall feeling of "minimal." If I had the graphic design -skills to make something impressive to look at I'd use them, but I don't, so -I wanted to stick with what I'm good at. - -I read a lot of programmers' blogs and one thing that annoys me is that I often -end up using [Safari Reader][] to make their sites pleasant to read. I want my -site to be pleasant to read *without* extra tools like Reader, so I took the easy -route and based the site's typography on Reader itself. - -Reader uses 18px [Palatino][] body text with a 25px line height, so that's what -I went with. I considered using [Georgia][] instead of Palatino, but -unfortunately Georgia has an absolutely terrible bold weight. I also kept the -width of the site small to mimic Reader. - -[Safari Reader]: http://www.apple.com/safari/whats-new.html#reader -[Palatino]: http://en.wikipedia.org/wiki/Palatino -[Georgia]: http://en.wikipedia.org/wiki/Georgia_(typeface) - -Creating a Vertical Rhythm --------------------------- - -Once I had a line height in place I could start working on a [vertical rhythm][]. -I used [Aardvark Legs][] as a base stylesheet and edited it to fit the new font -size and line height. - -Unfortunately the original post by Aardvark Legs' author seems to have -disappeared. In a nutshell: it's a CSS file that helps you set up a consistent -vertical rhythm. - -I tried to keep myself honest by including a tiny bit of Javascript in the -footer of the site that displays the vertical rhythm. If you click on it you -should see that everything lines up properly (except the footer itself, which -falls on a halfway point). - -For some entries the rhythm might not be perfect in non-Webkit browsers because -I had to use some [Webkit-specific styling][scrollbars] to fix the height of -scrollbars of wide `
` blocks.
-
-[vertical rhythm]: http://www.alistapart.com/articles/settingtypeontheweb/
-[aardvark legs]: http://aardvark.fecklessmind.com/
-[scrollbars]: http://webkit.org/blog/363/styling-scrollbars/
-
-Designing to a Dominant Seventh
--------------------------------
-
-In my spare time I play upright bass and teach blues dancing. I listen to a lot
-of jazz and blues music while doing both and one of the more common chords in
-those genres is a [dominant seventh][].
-
-What does this have to do with my site's design? Although I had a font size of
-18px for the body text I still needed to decide on sizes for headers and other
-site elements. Dominant seventh chords sound beautiful, so I used the notes in
-the chord to come up with these sizes.
-
-Here are the sizes I came up with (after rounding off to the nearest pixel):
-
-* Root: 18 pixels
-* Major third: 23 pixels
-* Perfect fifth: 27 pixels
-* Minor seventh: 32 pixels
-* Octave: 36 pixels
-* Major tenth: 45 pixels
-
-This gave me a variety of sizes to work with.  Here are a few places where
-I used them:
-
-* Base font size: root
-* `h1` elements: major tenth
-* `h2` elements: minor seventh
-* `h3` elements: major third
-* `h4` elements: root
-* "Logo" in the header: major third
-* Navigation links: root
-* Footer text: perfect fifth (an octave down)
-
-You might notice that I don't use a straight perfect fifth anywhere in the
-design.  There are two reasons for this.
-
-First: the perfect fifth and the line height I got from Reader are only two
-pixels apart, which would feel dissonant.
-
-Second: the perfect fifth is the most "generic" note in the chord, so
-eliminating it doesn't really change the feeling of the dominant seventh at
-all.
-
-Once I had my line height and header sizes I had plenty to work on.  I made
-images, blockquotes, code blocks, and everything else fit the rhythm. After
-that I decided to start playing with some other things.
-
-[dominant seventh]: http://en.wikipedia.org/wiki/Dominant_seventh_chord
-
-Removing Comments
------------------
-
-The first thing I did after I got the rhythm of the site down was to remove the
-[Disqus][] comments.
-
-I did this for two reasons.  The first and most "practical" reason was that
-they simply looked out of place.  It's hard to style content from a third-party
-site to look great with the rhythm of the site so I got rid of it altogether.
-
-The second reason is something I've learned from music and dancing: the most
-important notes are the ones you *don't* play.
-
-I'm usually pretty busy with full-time and freelance work and don't have enough
-time to properly respond to comments on the site.  Having comments here,
-then, is like having calls without responses -- it doesn't work very well.
-
-If someone wants to tell me something they can easily find me on
-[Twitter][twsl] and talk to me or discuss a post on [Hacker News][].
-
-[Disqus]: http://disqus.com/
-[twsl]: {{links.twsl}}
-[Hacker News]: http://news.ycombinator.com/
-
-Adding Applause with Flattr
----------------------------
-
-I've started using [Flattr][] to help show my appreciation to people that make
-good things on the internet.  Since I was redesigning the site I figured I'd
-include Flattr buttons on my blog posts.
-
-One reason was simple: it lets people give me money for the things I write.
-Another reason is a bit more altruistic: if more people see Flattr buttons
-around the web more people will sign up and help out creators.
-
-I like to think of Flattr as applause at a jazz show or a dance.  It doesn't
-really contribute anything to the conversation but it *does* let you easily
-show your appreciaton.
-
-The worst part of this was going through all my old blog posts and submitting
-them to Flattr.
-
-[Flattr]: http://flattr.com/
-
-Printing Nicely with Print Links
---------------------------------
-
-This is something that my old site also had but I never really talked about
-(except on [Forrst][]).
-
-If you print one of my site's pages you'll notice that any links inside
-a paragraph of content will show up in a list after the paragraph.  I think
-this makes it easier to refer to a printed copy because you can type in the
-URLs without having to visit my site again.
-
-To do this I use [a little bit of Javascript][printjs].  I *could* put the URLs
-directly after the links using just CSS, but I think that breaks up the content
-and makes it harder to read.
-
-[Forrst]: http://forrst.com/
-[printjs]: http://bitbucket.org/sjl/stevelosh/src/tip/media/js/print.js
-
-Showing Context with Scrolly Headers
-------------------------------------
-
-The last trick I pulled out of my sleeve was the "scrolly headers" to the left
-of the content, which you've probably already noticed.
-
-I tend to write fairly long blog posts and I think about their structure quite
-a bit.  Now that I'm using a much larger font I feel that it's too easy to get
-lost while reading them. If you switch away from the page for a bit and come
-back you might not remember where you were.
-
-To help fix this I wrote a bit of Javascript to display the top-level headers
-of post content to the left of the content.  I think this might help people
-keep their place more easily.
-
-Influences and Goals
---------------------
-
-While redesigning I had two main influences in mind:
-[Eivind Uggedal's site][uggedal] and [Simon Hørup Eskildsen's site][sirupsen].
-I hope they don't mind that I took a lot of elements from their designs to make
-my own.
-
-As I was making the new site I had a very simple goal: "make sure no one ever
-wants to use Safari Reader on my site."
-
-The design isn't particularly memorable (hell, it pretty much looks like you
-just used Safari Reader), but I don't think that's a problem.  I'm not
-a designer so I don't want my site to be remembered for its design. I want it
-to be remembered for the content, so I aimed for a design that highlights that
-content and gets out of the way.
-
-If you have comments, questions, or suggestions feel free to find me on
-[Twitter][twsl].
-
-[uggedal]: http://uggedal.com/
-[sirupsen]: http://sirupsen.com/
-
-{% endblock %}
diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2010/09/making-my-site-sing.markdown
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/content/blog/2010/09/making-my-site-sing.markdown	Fri Oct 07 12:48:36 2016 +0000
@@ -0,0 +1,226 @@
++++
+title = "Making My Site Sing"
+snip = "Designing with music."
+date = 2010-09-08T20:10:00Z
+draft = false
+
++++
+
+Every year or so I get the urge to redesign my site. It's only been seven
+months since the last time, so I guess working with [Nick Sergeant][] and
+[Ali Ali][] at [Dumbwaiter Design][] has made me less satisfied than usual
+with my previous design.
+
+I didn't switch frameworks this time — I'm still [using Hyde][] to generate
+a set of flat HTML files.  I only changed the structure and layout of the
+rendered site.
+
+I'm a programmer, not a designer, but I think it looks fairly good.  I wanted
+to write about how I went about designing the site because it might help other
+programmers.
+
+[Nick Sergeant]: http://nick.sg/
+[Ali Ali]: http://alialithinks.com/
+[Dumbwaiter Design]: http://dwaiter.com/
+[using Hyde]: /blog/2010/01/moving-from-django-to-hyde/
+
+{{% toc %}}
+
+Finding a Starting Point
+------------------------
+
+The first thing I had to do was come up with a basic idea for the style and
+typography of the site.
+
+I decided to go with my usual style: dark text on a light background, very few
+images, and an overall feeling of "minimal."  If I had the graphic design
+skills to make something impressive to look at I'd use them, but I don't, so
+I wanted to stick with what I'm good at.
+
+I read a lot of programmers' blogs and one thing that annoys me is that I often
+end up using [Safari Reader][] to make their sites pleasant to read.  I want my
+site to be pleasant to read *without* extra tools like Reader, so I took the easy
+route and based the site's typography on Reader itself.
+
+Reader uses 18px [Palatino][] body text with a 25px line height, so that's what
+I went with.  I considered using [Georgia][] instead of Palatino, but
+unfortunately Georgia has an absolutely terrible bold weight. I also kept the
+width of the site small to mimic Reader.
+
+[Safari Reader]: http://www.apple.com/safari/whats-new.html#reader
+[Palatino]: http://en.wikipedia.org/wiki/Palatino
+[Georgia]: http://en.wikipedia.org/wiki/Georgia_(typeface)
+
+Creating a Vertical Rhythm
+--------------------------
+
+Once I had a line height in place I could start working on a [vertical rhythm][].
+I used [Aardvark Legs][] as a base stylesheet and edited it to fit the new font
+size and line height.
+
+Unfortunately the original post by Aardvark Legs' author seems to have
+disappeared.  In a nutshell: it's a CSS file that helps you set up a consistent
+vertical rhythm.
+
+I tried to keep myself honest by including a tiny bit of Javascript in the
+footer of the site that displays the vertical rhythm. If you click on it you
+should see that everything lines up properly (except the footer itself, which
+falls on a halfway point).
+
+For some entries the rhythm might not be perfect in non-Webkit browsers because
+I had to use some [Webkit-specific styling][scrollbars] to fix the height of
+scrollbars of wide `
` blocks.
+
+[vertical rhythm]: http://www.alistapart.com/articles/settingtypeontheweb/
+[aardvark legs]: http://aardvark.fecklessmind.com/
+[scrollbars]: http://webkit.org/blog/363/styling-scrollbars/
+
+Designing to a Dominant Seventh
+-------------------------------
+
+In my spare time I play upright bass and teach blues dancing. I listen to a lot
+of jazz and blues music while doing both and one of the more common chords in
+those genres is a [dominant seventh][].
+
+What does this have to do with my site's design? Although I had a font size of
+18px for the body text I still needed to decide on sizes for headers and other
+site elements. Dominant seventh chords sound beautiful, so I used the notes in
+the chord to come up with these sizes.
+
+Here are the sizes I came up with (after rounding off to the nearest pixel):
+
+* Root: 18 pixels
+* Major third: 23 pixels
+* Perfect fifth: 27 pixels
+* Minor seventh: 32 pixels
+* Octave: 36 pixels
+* Major tenth: 45 pixels
+
+This gave me a variety of sizes to work with.  Here are a few places where
+I used them:
+
+* Base font size: root
+* `h1` elements: major tenth
+* `h2` elements: minor seventh
+* `h3` elements: major third
+* `h4` elements: root
+* "Logo" in the header: major third
+* Navigation links: root
+* Footer text: perfect fifth (an octave down)
+
+You might notice that I don't use a straight perfect fifth anywhere in the
+design.  There are two reasons for this.
+
+First: the perfect fifth and the line height I got from Reader are only two
+pixels apart, which would feel dissonant.
+
+Second: the perfect fifth is the most "generic" note in the chord, so
+eliminating it doesn't really change the feeling of the dominant seventh at
+all.
+
+Once I had my line height and header sizes I had plenty to work on.  I made
+images, blockquotes, code blocks, and everything else fit the rhythm. After
+that I decided to start playing with some other things.
+
+[dominant seventh]: http://en.wikipedia.org/wiki/Dominant_seventh_chord
+
+Removing Comments
+-----------------
+
+The first thing I did after I got the rhythm of the site down was to remove the
+[Disqus][] comments.
+
+I did this for two reasons.  The first and most "practical" reason was that
+they simply looked out of place.  It's hard to style content from a third-party
+site to look great with the rhythm of the site so I got rid of it altogether.
+
+The second reason is something I've learned from music and dancing: the most
+important notes are the ones you *don't* play.
+
+I'm usually pretty busy with full-time and freelance work and don't have enough
+time to properly respond to comments on the site.  Having comments here,
+then, is like having calls without responses — it doesn't work very well.
+
+If someone wants to tell me something they can easily find me on
+[Twitter][twsl] and talk to me or discuss a post on [Hacker News][].
+
+[Disqus]: http://disqus.com/
+[twsl]: {{links.twsl}}
+[Hacker News]: http://news.ycombinator.com/
+
+Adding Applause with Flattr
+---------------------------
+
+I've started using [Flattr][] to help show my appreciation to people that make
+good things on the internet.  Since I was redesigning the site I figured I'd
+include Flattr buttons on my blog posts.
+
+One reason was simple: it lets people give me money for the things I write.
+Another reason is a bit more altruistic: if more people see Flattr buttons
+around the web more people will sign up and help out creators.
+
+I like to think of Flattr as applause at a jazz show or a dance.  It doesn't
+really contribute anything to the conversation but it *does* let you easily
+show your appreciaton.
+
+The worst part of this was going through all my old blog posts and submitting
+them to Flattr.
+
+[Flattr]: http://flattr.com/
+
+Printing Nicely with Print Links
+--------------------------------
+
+This is something that my old site also had but I never really talked about
+(except on [Forrst][]).
+
+If you print one of my site's pages you'll notice that any links inside
+a paragraph of content will show up in a list after the paragraph.  I think
+this makes it easier to refer to a printed copy because you can type in the
+URLs without having to visit my site again.
+
+To do this I use [a little bit of Javascript][printjs].  I *could* put the URLs
+directly after the links using just CSS, but I think that breaks up the content
+and makes it harder to read.
+
+[Forrst]: http://forrst.com/
+[printjs]: http://bitbucket.org/sjl/stevelosh/src/tip/media/js/print.js
+
+Showing Context with Scrolly Headers
+------------------------------------
+
+The last trick I pulled out of my sleeve was the "scrolly headers" to the left
+of the content, which you've probably already noticed.
+
+I tend to write fairly long blog posts and I think about their structure quite
+a bit.  Now that I'm using a much larger font I feel that it's too easy to get
+lost while reading them. If you switch away from the page for a bit and come
+back you might not remember where you were.
+
+To help fix this I wrote a bit of Javascript to display the top-level headers
+of post content to the left of the content.  I think this might help people
+keep their place more easily.
+
+Influences and Goals
+--------------------
+
+While redesigning I had two main influences in mind:
+[Eivind Uggedal's site][uggedal] and [Simon Hørup Eskildsen's site][sirupsen].
+I hope they don't mind that I took a lot of elements from their designs to make
+my own.
+
+As I was making the new site I had a very simple goal: "make sure no one ever
+wants to use Safari Reader on my site."
+
+The design isn't particularly memorable (hell, it pretty much looks like you
+just used Safari Reader), but I don't think that's a problem.  I'm not
+a designer so I don't want my site to be remembered for its design. I want it
+to be remembered for the content, so I aimed for a design that highlights that
+content and gets out of the way.
+
+If you have comments, questions, or suggestions feel free to find me on
+[Twitter][twsl].
+
+[uggedal]: http://uggedal.com/
+[sirupsen]: http://sirupsen.com/
+
diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2010/11/keep-calm-and-carry-on.html
--- a/content/blog/2010/11/keep-calm-and-carry-on.html	Tue Sep 20 15:23:05 2016 +0000
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,248 +0,0 @@
-{% extends "_post.html" %}
-
-{% hyde
-    title: "Keep Calm and Carry On"
-    snip: "You don't always need to be sexy."
-    created: 2010-11-05 16:30:00
-%}
-
-{% block article %}
-
-I've been dancing quite a bit lately, both going to exchanges and teaching
-blues dancing with [Lady Luck Blues][llb]. I haven't written anything about
-dancing in quite a while, so I figured it was time for another blog post.
-
-This post will be about a particular idea (or if you prefer: "pet peeve") of
-mine about blues dancing today. I'm going to take a while to get to the point,
-but I think it's worth the reading.
-
-[llb]: http://ladyluckblues.com/
-
-[TOC]
-
-Saint James Infirmary
----------------------
-
-For this post I'm going to use a very popular song as an example: "Saint James
-Infirmary".  I'm sure almost every blues dancer has heard this song at some
-point (probably many times).
-
-There are *many* versions of this song around.  Here's one if you'd like to
-listen to it to refresh your memory:
-["Saint James Infirmary" by Snooks Eaglin on YouTube][sji].
-
-[sji]: http://www.youtube.com/watch?v=23VSDneTo60
-
-What is this Song About?
-------------------------
-
-Saint James Infirmary is a very old song.  There's a great overview of its
-history on [the Wikipedia page][wiki].
-
-Let's take a look at the lyrics and try to figure out what the song is trying
-to say.  The first verse goes (roughly) like this:
-
->I went down to Saint James infirmary,
->saw my baby there.
->She was stretched out on a long white table,
->so sweet, so cold, so fair. - -The first thing we learn about the song is that the singer's lover has died. -He goes to the morgue to view her body. We can already tell that this is not -going to be a happy song. - -Let's look at the next verse: - ->Let her go, let her go, god bless her
->wherever she may be.
->She can look this whole world all over
->and never find another man like me. - -In the first part of this verse the singer is accepting the fact that his lover -is dead, and wishing her well in any afterlife she may be in. The second part -is a bit less clear, but he seems to be telling us that there's no man on Earth -that loves (or, rather, "loved") her like he does. - -The last verse gets even darker (note: this verse's lyrics often vary quite -a bit between versions, but the idea is the same): - ->When I die you can bury me in straight laced shoes,
->a box-back coat and a Stetson hat.
->Put a twenty-dollar gold piece on my watch chain
->so all the boys will know I died standing pat. - -All of a sudden the singer is talking about his *own* death. What happened? - -The singer's lover died, he accepted her death, and now he gives instructions -on what to do when he dies. I'm sure some people will disagree, but to me it -definitely seems like he's contemplating suicide. - -Now that we've got a pretty clear idea of the "mood" of this song, I want to -talk about what bothers me about how many blues dancers seem to dance to it. - -[wiki]: http://en.wikipedia.org/wiki/Saint_James_Infirmary - -The Problem ------------ - -Blues dancing is commonly seen as a "sexy" dance. There's good reason for -this: many blues songs are lewd and suggestive, so being sexy as you dance fits -the music. - -The problem I see often is that dancers get comfortable in one "mood" of -dancing (usually "sexy") and don't bother to explore other ones. - -Almost without fail I see people dancing to Saint James Infirmary and trying -(often succeeding) to be sexy. They use lots of hip and body movement like -they do with other blues songs. - -I want to say something to them. - -**Stop it.** - -**You're doing it wrong.** - -**You there, doing the body roll: *stop*, god damn it!** - -Saint James Infirmary is *not* a "sexy" song. It's about death and suicide, -not hooking up! - -Would you ask someone for a date at their friend's funeral? No? Then why -would you dance like that to this song? It doesn't make sense and it's -completely inappropriate. - -We often talk about "musicality" in dance classes, but often it's just about -"hitting the breaks right." There's more to it than that. Reflecting the -music in your dancing isn't just about hitting the notes, it's about matching -the *mood* of the song too! - -The Solution ------------- - -Now is the time when I'm supposed to tell you how to fix things. I'm not the -best dancer out there, and it's hard to describe dancing in text, but I'll do -my best. - -If you don't agree with the specific things I mention that's completely cool -- -my goal is to at least get people *thinking* about these ideas, not to tell -them one specific way to implement them. - -### Followers - -The one major thing I'd like to tell followers is: "stop being sexy." There -are songs where that is completely appropriate, but this is not one of them. - -If you're only used to trying to be sexy, what can you do instead? - -The simple answer is: "just follow." Don't worry about adding styling if -you're not comfortable with it -- a solid follower is much more fun to dance -with than one that's trying to force a style she has no experience with. - -The more complicated answer is: "use styling that reflects the mood of the -song." Unfortunately I don't have much experience with following so I can't -really describe this. Take a private lesson with someone like -[Mike Legett][mike] or [Carsie Blanton][carsie] if you want to get a more -informed opinion. - -[mike]: http://www.mikethegirl.com/ -[carsie]: http://www.carsieblanton.com/ - -### Leaders - -As a leader, when I dance to this song I think about taking on one of two -personas: - -* The singer -- someone who has just lost a lover. -* A friend of the singer that is comforting him (or her, if my follower is - female). - -In both cases I try to eliminate any "swagger" or "bravado" from my styling -(not that I personally use much of that anyway). Funerals are not the place to -be an alpha male. - -If I'm taking on the singer's persona (someone that has lost a lover) I'll -usually dance in a "ballroomy" style. I'll use short movements (like muffled -sobs) punctuated by larger, sweeping movements (cries or wails). I'll (gasp) -slightly collapse my posture just a tiny bit to express the depression. - -If I choose the other case (comforting someone) I won't collapse my posture at -all. I'll try to represent the shoulder that someone would cry on when their -lover dies. I'll try to be strong, confident and solid, but not really -"manly." - -In both cases I'll almost always stay in close embrace for the whole song. -Whether you're comforting someone or being comforted, a hug is usually helpful -in dark times, so it feels appropriate to use close embrace. - -There are many other things you could do as a leader that would fit the song. -As long as you're conciously thinking of them and not just defaulting to -a style you're comfortable with, I'm happy. - -The Real Problem ----------------- - -Having said all that, I actually think the problem I've described is more of -a symptom, and there's a more fundamental problem with our community (and -culture in general) today: - -**People don't simply *listen* to music any more.** - -They hear music while dancing, or put on headphones while doing homework, or -turn on the radios in their cars, but they never just sit down to *listen* to -a song without doing anything else. - -Many years ago, families would sit around the radio during the evenings and -just listen to the music. Today the radio has been replaced by television, so -we no longer even have those hours. - -Dancers may be dancing provocatively to Saint James Infirmary because *they -don't even realize it's a sad song*! Even though they've heard it many times -it never really registers, because they're always focusing on something else -when they hear it. - -The Real Solution ------------------ - -This root problem has a much more clear solution: **listen to music, damn it!** - -Here's the basic idea: - -* Find a good album. Ask around, there are plenty out there. -* Find a good pair of headphones. Your iPod earbuds do not count. Borrow a pair - if necessary. -* Turn off the television, put away books, turn off your phone and your laptop - (or at least quit everything on your laptop if you're using it to play the music). -* Start the album. - -Now that you're finally listening, what should you be trying to hear? Here are -a few suggestions: - -* Try just listening to a couple of songs. Take in the lyrics (if they have - them) and the overall "mood" of the instruments. -* Listen to the energy of the songs. Jazz and blues musicians will usually make - the energy rise and fall throughout the song. -* Pick a single instrument and listen to it for an entire song. Try not to let - your mind wander -- really focus on just one instrument. This is where a decent - pair of headphones will really help. -* Pick a different instrument and listen to it for an entire song. Try to notice - differences in how the two instruments play. For example, a bassist will often - be playing for the entire song, while a horn player will frequently stop playing - while others are soloing. - -Once you've listened to the entire album, without stopping, get a notebook and -write down some of the things you noticed. You don't have to have any amazing -revelations -- the point is to make yourself put into words what you're -hearing. - -Do this at least once or twice a week for a few months. - -Putting down these thoughts on paper will help you wrap your head around music -when you hear it during a dance. After a while you'll start hearing structure -and themes in the music and can adjust your dancing to match them. - -Music is the foundation of dancing, so the more we listen the better our -dancing will be. - -We'll all become better dancers if we just stop and *listen*. - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2010/11/keep-calm-and-carry-on.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2010/11/keep-calm-and-carry-on.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,245 @@ ++++ +title = "Keep Calm and Carry On" +snip = "You don't always need to be sexy." +date = 2010-11-05T16:30:00Z +draft = false + ++++ + +I've been dancing quite a bit lately, both going to exchanges and teaching +blues dancing with [Lady Luck Blues][llb]. I haven't written anything about +dancing in quite a while, so I figured it was time for another blog post. + +This post will be about a particular idea (or if you prefer: "pet peeve") of +mine about blues dancing today. I'm going to take a while to get to the point, +but I think it's worth the reading. + +[llb]: http://ladyluckblues.com/ + +{{% toc %}} + +Saint James Infirmary +--------------------- + +For this post I'm going to use a very popular song as an example: "Saint James +Infirmary". I'm sure almost every blues dancer has heard this song at some +point (probably many times). + +There are *many* versions of this song around. Here's one if you'd like to +listen to it to refresh your memory: +["Saint James Infirmary" by Snooks Eaglin on YouTube][sji]. + +[sji]: http://www.youtube.com/watch?v=23VSDneTo60 + +What is this Song About? +------------------------ + +Saint James Infirmary is a very old song. There's a great overview of its +history on [the Wikipedia page][wiki]. + +Let's take a look at the lyrics and try to figure out what the song is trying +to say. The first verse goes (roughly) like this: + +>I went down to Saint James infirmary,
+>saw my baby there.
+>She was stretched out on a long white table,
+>so sweet, so cold, so fair. + +The first thing we learn about the song is that the singer's lover has died. +He goes to the morgue to view her body. We can already tell that this is not +going to be a happy song. + +Let's look at the next verse: + +>Let her go, let her go, god bless her
+>wherever she may be.
+>She can look this whole world all over
+>and never find another man like me. + +In the first part of this verse the singer is accepting the fact that his lover +is dead, and wishing her well in any afterlife she may be in. The second part +is a bit less clear, but he seems to be telling us that there's no man on Earth +that loves (or, rather, "loved") her like he does. + +The last verse gets even darker (note: this verse's lyrics often vary quite +a bit between versions, but the idea is the same): + +>When I die you can bury me in straight laced shoes,
+>a box-back coat and a Stetson hat.
+>Put a twenty-dollar gold piece on my watch chain
+>so all the boys will know I died standing pat. + +All of a sudden the singer is talking about his *own* death. What happened? + +The singer's lover died, he accepted her death, and now he gives instructions +on what to do when he dies. I'm sure some people will disagree, but to me it +definitely seems like he's contemplating suicide. + +Now that we've got a pretty clear idea of the "mood" of this song, I want to +talk about what bothers me about how many blues dancers seem to dance to it. + +[wiki]: http://en.wikipedia.org/wiki/Saint_James_Infirmary + +The Problem +----------- + +Blues dancing is commonly seen as a "sexy" dance. There's good reason for +this: many blues songs are lewd and suggestive, so being sexy as you dance fits +the music. + +The problem I see often is that dancers get comfortable in one "mood" of +dancing (usually "sexy") and don't bother to explore other ones. + +Almost without fail I see people dancing to Saint James Infirmary and trying +(often succeeding) to be sexy. They use lots of hip and body movement like +they do with other blues songs. + +I want to say something to them. + +**Stop it.** + +**You're doing it wrong.** + +**You there, doing the body roll: *stop*, god damn it!** + +Saint James Infirmary is *not* a "sexy" song. It's about death and suicide, +not hooking up! + +Would you ask someone for a date at their friend's funeral? No? Then why +would you dance like that to this song? It doesn't make sense and it's +completely inappropriate. + +We often talk about "musicality" in dance classes, but often it's just about +"hitting the breaks right." There's more to it than that. Reflecting the +music in your dancing isn't just about hitting the notes, it's about matching +the *mood* of the song too! + +The Solution +------------ + +Now is the time when I'm supposed to tell you how to fix things. I'm not the +best dancer out there, and it's hard to describe dancing in text, but I'll do +my best. + +If you don't agree with the specific things I mention that's completely cool -- +my goal is to at least get people *thinking* about these ideas, not to tell +them one specific way to implement them. + +### Followers + +The one major thing I'd like to tell followers is: "stop being sexy." There +are songs where that is completely appropriate, but this is not one of them. + +If you're only used to trying to be sexy, what can you do instead? + +The simple answer is: "just follow." Don't worry about adding styling if +you're not comfortable with it — a solid follower is much more fun to dance +with than one that's trying to force a style she has no experience with. + +The more complicated answer is: "use styling that reflects the mood of the +song." Unfortunately I don't have much experience with following so I can't +really describe this. Take a private lesson with someone like +[Mike Legett][mike] or [Carsie Blanton][carsie] if you want to get a more +informed opinion. + +[mike]: http://www.mikethegirl.com/ +[carsie]: http://www.carsieblanton.com/ + +### Leaders + +As a leader, when I dance to this song I think about taking on one of two +personas: + +* The singer — someone who has just lost a lover. +* A friend of the singer that is comforting him (or her, if my follower is + female). + +In both cases I try to eliminate any "swagger" or "bravado" from my styling +(not that I personally use much of that anyway). Funerals are not the place to +be an alpha male. + +If I'm taking on the singer's persona (someone that has lost a lover) I'll +usually dance in a "ballroomy" style. I'll use short movements (like muffled +sobs) punctuated by larger, sweeping movements (cries or wails). I'll (gasp) +slightly collapse my posture just a tiny bit to express the depression. + +If I choose the other case (comforting someone) I won't collapse my posture at +all. I'll try to represent the shoulder that someone would cry on when their +lover dies. I'll try to be strong, confident and solid, but not really +"manly." + +In both cases I'll almost always stay in close embrace for the whole song. +Whether you're comforting someone or being comforted, a hug is usually helpful +in dark times, so it feels appropriate to use close embrace. + +There are many other things you could do as a leader that would fit the song. +As long as you're conciously thinking of them and not just defaulting to +a style you're comfortable with, I'm happy. + +The Real Problem +---------------- + +Having said all that, I actually think the problem I've described is more of +a symptom, and there's a more fundamental problem with our community (and +culture in general) today: + +**People don't simply *listen* to music any more.** + +They hear music while dancing, or put on headphones while doing homework, or +turn on the radios in their cars, but they never just sit down to *listen* to +a song without doing anything else. + +Many years ago, families would sit around the radio during the evenings and +just listen to the music. Today the radio has been replaced by television, so +we no longer even have those hours. + +Dancers may be dancing provocatively to Saint James Infirmary because *they +don't even realize it's a sad song*! Even though they've heard it many times +it never really registers, because they're always focusing on something else +when they hear it. + +The Real Solution +----------------- + +This root problem has a much more clear solution: **listen to music, damn it!** + +Here's the basic idea: + +* Find a good album. Ask around, there are plenty out there. +* Find a good pair of headphones. Your iPod earbuds do not count. Borrow a pair + if necessary. +* Turn off the television, put away books, turn off your phone and your laptop + (or at least quit everything on your laptop if you're using it to play the music). +* Start the album. + +Now that you're finally listening, what should you be trying to hear? Here are +a few suggestions: + +* Try just listening to a couple of songs. Take in the lyrics (if they have + them) and the overall "mood" of the instruments. +* Listen to the energy of the songs. Jazz and blues musicians will usually make + the energy rise and fall throughout the song. +* Pick a single instrument and listen to it for an entire song. Try not to let + your mind wander — really focus on just one instrument. This is where a decent + pair of headphones will really help. +* Pick a different instrument and listen to it for an entire song. Try to notice + differences in how the two instruments play. For example, a bassist will often + be playing for the entire song, while a horn player will frequently stop playing + while others are soloing. + +Once you've listened to the entire album, without stopping, get a notebook and +write down some of the things you noticed. You don't have to have any amazing +revelations — the point is to make yourself put into words what you're +hearing. + +Do this at least once or twice a week for a few months. + +Putting down these thoughts on paper will help you wrap your head around music +when you hear it during a dance. After a while you'll start hearing structure +and themes in the music and can adjust your dancing to match them. + +Music is the foundation of dancing, so the more we listen the better our +dancing will be. + +We'll all become better dancers if we just stop and *listen*. + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2011/05/on-learning-and-teaching.html --- a/content/blog/2011/05/on-learning-and-teaching.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,258 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "On Learning and Teaching" - snip: "Learning one thing isn't enough." - created: 2011-05-22 17:00:00 - %} - - {% block article %} - -I recently read a blog entry by [Zack Kurmas][] called "[The deep end of the -pool][]". In it he talks about why he thinks some students succeed in introductory -Computer Science classes while others fail. If you're a programmer and haven't read -it, stop right now and read it. - -In a nutshell: I think he's pretty much correct. I wanted to write this post because -I think I have a solution to the problem that might work, at least for some people. - -[TOC] - -[Zack Kurmas]: http://www.cis.gvsu.edu/~kurmasz/ -[The deep end of the pool]: http://spin.atomicobject.com/2011/05/17/the-deep-end-of-the-pool/ - -Background ----------- - -I'm a computer programmer by trade. I went to [RIT][] for five years to get my -degree in Computer Science and before that I was working with toy programs during -high school. - -While I was at RIT I started going to [Swing Dance Club][]. I learned to swing dance -there about eight years ago. A few years later I started I began teaching Swing -Dance Club with some of my friends, and about a year ago I started teaching Blues -Dancing with [Lady Luck Blues][]. - -I have a couple of other semi-serious hobbies that I've taken classes in as well: -photography and bass playing. - -To summarize: I've had experience in a bunch of different areas, and I've been -teaching one of them for a while. - -**Disclaimer:** one thing I've never studied is teaching itself, so feel free to take -everything I say about it with a large grain of salt. I haven't done any hard -research and I could be completely wrong. - -[RIT]: http://rit.edu/ -[Swing Dance Club]: http://www.rit.edu/sg/swing/ -[Lady Luck Blues]: http://ladyluckblues.com/ - -Similarities in Dancing and Programming ---------------- - -In his article, Zack talks about programming syntax being a stumbling block for -beginners. I think he's very right and I think there's a similar concept in dancing. - -### Programming Syntax - -If you're a programmer you can skip this section. - -For the non-programmers: "syntax" is the combination of letters, numbers and symbols -that you write to create computer programs. For example, look at the following two -bits of text: - - if x == 2: - print "Hello" - x = 10 - - if x == 2: - print 'Hello" - x = 10 - -Notice the difference? The first snippet says `"Hello"` while the second says -`'Hello"`. Notice how the first uses two double quotes (`"`) while the second uses -one double and one single quote (`'`). - -The first bit of code will work, but the second one won't. Little differences like -these might not seem like a big deal, but to a computer they're as different as -`abcdefg` and `o34%^8@@.com`. - -### Dancing Footwork - -If you're a dancer you can skip this section. - -For the non-dancers: "footwork" is the pattern of steps you make when you dance. For -example, in Lindy Hop (a style of swing dancing) the basic footwork looks like this: - - Beats: 1 2 3 4 5 6 7 8 - Steps: x x x xx x x x xx - -For East Coast Swing, the basic footwork looks like this: - - Beats: 1 2 3 4 5 6 - Steps: x x x xx x xx - -Notice the differences? Lindy Hop footwork is based on an eight beat pattern, while -East Coast footwork is based on six beats. The patterns also have the "extra steps" -in different places. - -When you're dancing with a partner you both need to be doing the same footwork, -otherwise you can't really dance together. - -### Footwork is Syntax and Syntax is Footwork - -When you're learning both programming and dancing there are lots of deep, high-level -concepts you can tackle. For programming you have things like object orientation, -recursion, and API design. In dancing you have things like musicality, listening to -your follower, and negative space. - -However: in my experience teaching dancing you need to learn and understand the -footwork before you can learn the higher concepts. If you're still thinking about -where to put your feet when you're dancing you won't have the spare brainpower to -think about those other concepts. - -In Zack's post he says the same thing about programming. If you're still struggling -with understanding the syntax of programs you can't simultaneously pay attention to -the higher concepts that are being taught. I've never taught programming but my -personal observations of programmers make me think he's right. - -To be clear: there aren't just two levels (low and high). There are many -levels/layers, each of which builds on the last. Also: the layers are not linear -- -learning concept A might allow you to learn either B or C. - -However: the same idea applies most of the time. You need to learn and internalize -lower levels before moving on to the higher ones. - -Plateaus in Learning ------------------- - -One thing I've noticed while learning dancing, programming, photography, and other -skills is that my learning seems to progress quickly at times and slowly at others. - -When I'm learning quickly everything is great. I feel good about myself and my -abilities and I throw myself into learning as much as I can. - -On the other hand when I'm learning slowly (hitting a "plateau") I often get -frustrated or discouraged. Nothing I try seems to speed up my ascent out of these -plateaus. It seems like the only thing that works is time and practice of what -I already know. - -I think the reason for this is that once I learn a bunch of new concepts my brain -needs some time to process them. I can learn the concepts and use them if -I concentrate, but it's not effortless. I think the reason I finally start learning -quickly again after some time is that I've internalized the previous concepts and now -I can move on to more difficult ones which build on the previous ones. - -The first plateau of programming is the syntax and the first plateau of dancing is -footwork. The bad part about this is that dancing when you only know footwork or -programming when you only know syntax isn't much fun. You can't do all of the most -interesting things that make these skills so rewarding. - -Getting Through Plateaus ------------------------- - -In my own experience the hardest part about teaching is getting students through the -plateaus. Because it just takes time and practice to process what they've already -learned there's not much I can do to help them. Students take more advanced classes -but don't really grab them until they're fluent in the things they've previously -learned. - -As a teacher, this is incredibly frustrating. I want to teach my students what -I know, but I know they won't absorb it until they've absorbed the previous lessons. - -### The Lie and the Solution - -Here's the kicker, and what I think might provide the solution to this problem: **I -lied when I said that the only things that help me through plateaus are time and -practice.** - -There's one more thing that helps me get through plateaus: **connecting the concepts -I'm trying to internalize with concepts from other skills that I already know.** - -An example of this that I've written about before is [negative space][]. When I was -learning blues dancing I realized that "stopping your movement" was simply -a different expression of the concept of "negative space" in photography. - -I had already struggled through internalizing the concept of negative space in -photography, so when I connected this with dancing it helped make things "click" -together and made the plateau shorter. - -[negative space]: http://stevelosh.com/blog/2008/08/negative-space-dancing/ - -### The Problem with the Solution - -In my own life this solution is great. - -I have a bunch of hobbies and all of them mesh together in different ways. Sometimes -I can connect what I know in one to what I know in another and get through these -plateaus more quickly. When I can't do that I can focus on one of my other hobbies -while practicing another, which reduces the frustration of not progressing faster. - -When teaching students, however, this isn't a trick I can use effectively. - -I can't really tell a student: "you need to find something else to focus on for -a while while you internalize what I've taught you." If they want to learn dancing, -they'll feel like taking more dance classes is the way to go. They're paying me to -teach them so I obviously have to try to teach them. - -I sometimes try to teach the "analogies" between skills that have helped me. -Sometimes this works well -- the "Negative Space in Dancing" class that my partner -and I have taught seems to go over well with most of the students. - -The problem is that everyone has different skills. Trying to connect a concept in -programming to a concept in photography is futile if the student doesn't understand -the photographic concept. - -### The Solution to the Problem with the Solution - -Disclaimer: once again, I've never studied teaching itself, so this might not be -a new idea and/or I could be completely wrong about this. - -I've thought a long time about a way to fix this. For students that are simply -taking a class here and there with me, I haven't found a good solution. - -However, for students enrolled in a longer series of classes, like college students -studying a major, I think there might be a way to use this idea (linking concepts -from disparate skills) to help them learn. - -When I went to RIT I didn't just take Computer Science classes. I had to take a few -science classes, a few math classes, a few history classes, and so on. I believe -other colleges work on the same system: you take many classes in your major and one -or two classes in everything else. The idea is that you become a "well-rounded" -person. - -The bad part about this process is that one or two classes is not enough to get to -one of these "plateaus", struggle through it, and continue on. Therefore you don't -have time to internalize the skills you learn, which means that you can't connect -them to concepts in your major. - -I'm also not sure how effective these random classes are at making students -"well-rounded". If you don't get far enough to internalize the skills you learn in -a class are they really useful to you once you're done with those classes? - -I think the way around this is to ditch the "one major and a tiny bit of everything -else" process and move to a "one major and two or three minors" process. - -This will help students past the plateaus that discourage them, because they'll have -learned enough in their "minors" to be able to connect concepts to those in their -major. Teachers might not be able to teach specific analogies because the students -will all have different minors, but each student will be able to connect the concepts -in their majors on their own to whatever they happen to know from their minors. - -This process also has another benefit: students will leave college retaining more -skills that they would have otherwise. A programming student could leave college -with a real, effective amount of knowledge in music; a writing major could leave with -a real, effective amount of knowledge in biology, and so on. - -Conclusion ----------- - -I'm not naive enough to think that any college is going to change their entire -curriculum based on some random guy's blog post. - -My intended audience for this post is not colleges, but people that want to learn -skills faster and more effectively. I know that there are a lot of people out there -hungry for knowledge and I hope that some of them can benefit from the ideas I've -given, learn faster, and make the world a better place. - - {% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2011/05/on-learning-and-teaching.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2011/05/on-learning-and-teaching.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,255 @@ ++++ +title = "On Learning and Teaching" +snip = "Learning one thing isn't enough." +date = 2011-05-22T17:00:00Z +draft = false + ++++ + +I recently read a blog entry by [Zack Kurmas][] called "[The deep end of the +pool][]". In it he talks about why he thinks some students succeed in introductory +Computer Science classes while others fail. If you're a programmer and haven't read +it, stop right now and read it. + +In a nutshell: I think he's pretty much correct. I wanted to write this post because +I think I have a solution to the problem that might work, at least for some people. + +{{% toc %}} + +[Zack Kurmas]: http://www.cis.gvsu.edu/~kurmasz/ +[The deep end of the pool]: http://spin.atomicobject.com/2011/05/17/the-deep-end-of-the-pool/ + +Background +---------- + +I'm a computer programmer by trade. I went to [RIT][] for five years to get my +degree in Computer Science and before that I was working with toy programs during +high school. + +While I was at RIT I started going to [Swing Dance Club][]. I learned to swing dance +there about eight years ago. A few years later I started I began teaching Swing +Dance Club with some of my friends, and about a year ago I started teaching Blues +Dancing with [Lady Luck Blues][]. + +I have a couple of other semi-serious hobbies that I've taken classes in as well: +photography and bass playing. + +To summarize: I've had experience in a bunch of different areas, and I've been +teaching one of them for a while. + +**Disclaimer:** one thing I've never studied is teaching itself, so feel free to take +everything I say about it with a large grain of salt. I haven't done any hard +research and I could be completely wrong. + +[RIT]: http://rit.edu/ +[Swing Dance Club]: http://www.rit.edu/sg/swing/ +[Lady Luck Blues]: http://ladyluckblues.com/ + +Similarities in Dancing and Programming +--------------- + +In his article, Zack talks about programming syntax being a stumbling block for +beginners. I think he's very right and I think there's a similar concept in dancing. + +### Programming Syntax + +If you're a programmer you can skip this section. + +For the non-programmers: "syntax" is the combination of letters, numbers and symbols +that you write to create computer programs. For example, look at the following two +bits of text: + + if x == 2: + print "Hello" + x = 10 + + if x == 2: + print 'Hello" + x = 10 + +Notice the difference? The first snippet says `"Hello"` while the second says +`'Hello"`. Notice how the first uses two double quotes (`"`) while the second uses +one double and one single quote (`'`). + +The first bit of code will work, but the second one won't. Little differences like +these might not seem like a big deal, but to a computer they're as different as +`abcdefg` and `o34%^8@@.com`. + +### Dancing Footwork + +If you're a dancer you can skip this section. + +For the non-dancers: "footwork" is the pattern of steps you make when you dance. For +example, in Lindy Hop (a style of swing dancing) the basic footwork looks like this: + + Beats: 1 2 3 4 5 6 7 8 + Steps: x x x xx x x x xx + +For East Coast Swing, the basic footwork looks like this: + + Beats: 1 2 3 4 5 6 + Steps: x x x xx x xx + +Notice the differences? Lindy Hop footwork is based on an eight beat pattern, while +East Coast footwork is based on six beats. The patterns also have the "extra steps" +in different places. + +When you're dancing with a partner you both need to be doing the same footwork, +otherwise you can't really dance together. + +### Footwork is Syntax and Syntax is Footwork + +When you're learning both programming and dancing there are lots of deep, high-level +concepts you can tackle. For programming you have things like object orientation, +recursion, and API design. In dancing you have things like musicality, listening to +your follower, and negative space. + +However: in my experience teaching dancing you need to learn and understand the +footwork before you can learn the higher concepts. If you're still thinking about +where to put your feet when you're dancing you won't have the spare brainpower to +think about those other concepts. + +In Zack's post he says the same thing about programming. If you're still struggling +with understanding the syntax of programs you can't simultaneously pay attention to +the higher concepts that are being taught. I've never taught programming but my +personal observations of programmers make me think he's right. + +To be clear: there aren't just two levels (low and high). There are many +levels/layers, each of which builds on the last. Also: the layers are not linear -- +learning concept A might allow you to learn either B or C. + +However: the same idea applies most of the time. You need to learn and internalize +lower levels before moving on to the higher ones. + +Plateaus in Learning +------------------ + +One thing I've noticed while learning dancing, programming, photography, and other +skills is that my learning seems to progress quickly at times and slowly at others. + +When I'm learning quickly everything is great. I feel good about myself and my +abilities and I throw myself into learning as much as I can. + +On the other hand when I'm learning slowly (hitting a "plateau") I often get +frustrated or discouraged. Nothing I try seems to speed up my ascent out of these +plateaus. It seems like the only thing that works is time and practice of what +I already know. + +I think the reason for this is that once I learn a bunch of new concepts my brain +needs some time to process them. I can learn the concepts and use them if +I concentrate, but it's not effortless. I think the reason I finally start learning +quickly again after some time is that I've internalized the previous concepts and now +I can move on to more difficult ones which build on the previous ones. + +The first plateau of programming is the syntax and the first plateau of dancing is +footwork. The bad part about this is that dancing when you only know footwork or +programming when you only know syntax isn't much fun. You can't do all of the most +interesting things that make these skills so rewarding. + +Getting Through Plateaus +------------------------ + +In my own experience the hardest part about teaching is getting students through the +plateaus. Because it just takes time and practice to process what they've already +learned there's not much I can do to help them. Students take more advanced classes +but don't really grab them until they're fluent in the things they've previously +learned. + +As a teacher, this is incredibly frustrating. I want to teach my students what +I know, but I know they won't absorb it until they've absorbed the previous lessons. + +### The Lie and the Solution + +Here's the kicker, and what I think might provide the solution to this problem: **I +lied when I said that the only things that help me through plateaus are time and +practice.** + +There's one more thing that helps me get through plateaus: **connecting the concepts +I'm trying to internalize with concepts from other skills that I already know.** + +An example of this that I've written about before is [negative space][]. When I was +learning blues dancing I realized that "stopping your movement" was simply +a different expression of the concept of "negative space" in photography. + +I had already struggled through internalizing the concept of negative space in +photography, so when I connected this with dancing it helped make things "click" +together and made the plateau shorter. + +[negative space]: http://stevelosh.com/blog/2008/08/negative-space-dancing/ + +### The Problem with the Solution + +In my own life this solution is great. + +I have a bunch of hobbies and all of them mesh together in different ways. Sometimes +I can connect what I know in one to what I know in another and get through these +plateaus more quickly. When I can't do that I can focus on one of my other hobbies +while practicing another, which reduces the frustration of not progressing faster. + +When teaching students, however, this isn't a trick I can use effectively. + +I can't really tell a student: "you need to find something else to focus on for +a while while you internalize what I've taught you." If they want to learn dancing, +they'll feel like taking more dance classes is the way to go. They're paying me to +teach them so I obviously have to try to teach them. + +I sometimes try to teach the "analogies" between skills that have helped me. +Sometimes this works well — the "Negative Space in Dancing" class that my partner +and I have taught seems to go over well with most of the students. + +The problem is that everyone has different skills. Trying to connect a concept in +programming to a concept in photography is futile if the student doesn't understand +the photographic concept. + +### The Solution to the Problem with the Solution + +Disclaimer: once again, I've never studied teaching itself, so this might not be +a new idea and/or I could be completely wrong about this. + +I've thought a long time about a way to fix this. For students that are simply +taking a class here and there with me, I haven't found a good solution. + +However, for students enrolled in a longer series of classes, like college students +studying a major, I think there might be a way to use this idea (linking concepts +from disparate skills) to help them learn. + +When I went to RIT I didn't just take Computer Science classes. I had to take a few +science classes, a few math classes, a few history classes, and so on. I believe +other colleges work on the same system: you take many classes in your major and one +or two classes in everything else. The idea is that you become a "well-rounded" +person. + +The bad part about this process is that one or two classes is not enough to get to +one of these "plateaus", struggle through it, and continue on. Therefore you don't +have time to internalize the skills you learn, which means that you can't connect +them to concepts in your major. + +I'm also not sure how effective these random classes are at making students +"well-rounded". If you don't get far enough to internalize the skills you learn in +a class are they really useful to you once you're done with those classes? + +I think the way around this is to ditch the "one major and a tiny bit of everything +else" process and move to a "one major and two or three minors" process. + +This will help students past the plateaus that discourage them, because they'll have +learned enough in their "minors" to be able to connect concepts to those in their +major. Teachers might not be able to teach specific analogies because the students +will all have different minors, but each student will be able to connect the concepts +in their majors on their own to whatever they happen to know from their minors. + +This process also has another benefit: students will leave college retaining more +skills that they would have otherwise. A programming student could leave college +with a real, effective amount of knowledge in music; a writing major could leave with +a real, effective amount of knowledge in biology, and so on. + +Conclusion +---------- + +I'm not naive enough to think that any college is going to change their entire +curriculum based on some random guy's blog post. + +My intended audience for this post is not colleges, but people that want to learn +skills faster and more effectively. I know that there are a lot of people out there +hungry for knowledge and I hope that some of them can benefit from the ideas I've +given, learn faster, and make the world a better place. + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2011/05/paper-free.html --- a/content/blog/2011/05/paper-free.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,233 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "Going Paper-Free for $220" - snip: "It feels like the future!" - created: 2011-05-26 13:44:00 - %} - - {% block article %} - -It's 2011. Personal computers have been around and popular for well over a decade -now, and yet we still have to deal with a huge amount of physical paper. - -I've been wanting to go paper-free for a long time now. The advantages are obvious: - -- Paper takes up physical space in our homes that digital files don't. -- Digital files, if properly encrypted, are far more secure than sheets of paper that - could be stolen. -- Digital files can be searched in an instant, while papers have to be laboriously - sorted through. -- Digital files can be backed up perfectly and easily. - -After reading [this article][] I was psyched to scan and shred all the boxes of paper -sitting in my apartment, but the $420+ price tag was hard to swallow. I started -looking around for other options. - -[this article]: http://ryanwaggoner.com/2010/11/how-i-filled-two-dumpsters-and-went-paperless-with-the-fujitsu-scansnap-s1500/ - -Here are the requirements I have for any paper-free system: - -- The scanned files need to be OCR'ed so I can search them easily. I'm too lazy to - categorize and tag files manually. -- I need to be able to scan files anywhere. If I'm out at dinner I want to be able - to snap a picture of my receipt and tear it up right there. -- No "cloud" services allowed for unencrypted important documents. I simply don't - trust Google/Dropbox/etc enough to put my bank statements and such there. -- Files need to be backed up securely in case my apartment burns down. -- The entire process needs to be automated as much as possible, otherwise I'll get - lazy and not scan things. - -It's taken me a while, but I've finally got a system I'm happy with. This post will -describe each part and how they fit together. The total cost is about $220, $160 of -which is for a physical scanner. - -Note: I use OS X and an iPhone, so this post will focus on that platform. However, -the important pieces of software will run on Windows and I'm sure there are -Windows/Android equivalents to the other pieces. - -[TOC] - -Scanning at Home ----------------- - -The first step to becoming paper-free is obviously scanning your documents. There -are a lot of scanners out there, some more expensive than others. I eventually -settled on a [Doxie][] for $160. - -[Doxie]: http://www.getdoxie.com/ - -I chose the Doxie because: - -- It's compact. -- It runs with a single USB cable. -- It's cross-platform. -- Its software has a great, polished UI. -- It has a "multiple-function button" that lets you control it without the - mouse/keyboard. - -The first, second, and last points mean that (with a USB extension cable) I can scan -documents while sitting on the couch and watching Netflix, which is critical for lazy -people like me. - -I set Doxie to save scans on my Desktop. The scanning process is pretty simple so -I won't describe it here. Check out Doxie's documentation for more information. - -**Note:** When I first received my Doxie and tried to calibrate it, it simply made -a grinding noise and wouldn't feed the paper. I emailed their tech support and -within half an hour I got a response back saying they were shipping me a replacement -immediately. - -When I got the replacement it worked like a charm. Their customer service was so -great that I'd still recommend the Doxie even though my first one was a dud. - -Scanning on the Go ------------------- - -As I mentioned before, I want to be able to scan things while out and about with my -iPhone. There are a bunch of iPhone document-scanning apps out there. I settled on -[JotNot][] for $7 because it has a decent UI and supports multiple-page PDFs. - -JotNot's UI is pretty easy to get the hang of so I won't go over it here. - -Once I finish scanning something I send the PDF to a [Dropbox][] folder called -"JotNot". - -I know I said in my requirements that "cloud" services weren't allowed, but I make an -exception for non-critical things that I'd be scanning with my phone. I don't care if -Dropbox knows how much I spent on dinner. - -[JotNot]: http://itunes.apple.com/us/app/jotnot-scanner-pro/id307868751?mt=8 -[Dropbox]: http://www.dropbox.com/ - -OCR'ing Scanned Documents -------------------------- - -The next step is to run the scanned PDFs through an OCR program so they can be -searched with Spotlight. - -I looked at a lot of OCR software and finally settled on [PDF OCR X][] for $30. It -has a simple interface, does a pretty good job at OCR'ing, has a free version so -I could try it out, and is cross-platform. - -Using it is simple: you drag a PDF onto the app and select your desired settings -(make sure to choose "searchable PDF" as the output format). The app will think for -a while and then create a new PDF next to the old one with the searchable text -embedded. - -Once you've done this once you should go into the preferences and change it to -non-interactive mode so that it won't prompt you for the settings every time you use -it. - -[PDF OCR X]: http://solutions.weblite.ca/pdfocrx/ - -Gluing Everything Together --------------------------- - -So far we've got two folders with scanned PDFs and a method for OCR'ing them. The -next step is to automate the process. - -I use an app called [Hazel][] to do this. It's $21 for a license and well worth it. -We'll set up four rules to make our lives easier. - -Before we start we need to create two folders somewhere (you can name them whatever -you like): - -- Pending OCR: A folder to hold documents that are waiting to be OCR'ed. -- Dead Trees: A folder to hold the final, OCR'ed versions of our documents. - -The first rule watches the Desktop for scans from Doxie. Any files placed on the -Desktop whose name starts with "Doxie Doc" will be renamed to include the current -date and time, and then moved to the "Pending OCR" folder. - -![Rule 1 Screenshot](/media/images{{ parent_url }}/rules-1-doxie.png "Rule 1") - -**Note:**: you'll need to click the `date created` bubble and then "Edit Date" to get -the time as well as the date into the filename. - -The second rule watches the "JotNot" folder for scans from the iPhone app. Any PDFs -that appear in here (i.e. that are synced down from Dropbox) will be moved to the -"Pending OCR" folder. We don't need to rename them like we did with the Doxie scans -because JotNot already includes the date and time of scans in the filenames by -default. - -![Rule 2 Screenshot](/media/images{{ parent_url }}/rules-2-jotnot.png "Rule 2") - -Now that we've got all of our scans going into the same folder (with unique names) we -can set up a rule to OCR them. The third rule watches the "Pending OCR" folder for -PDFs. When a PDF lands in the folder it will be moved to its final destination -folder ("Dead Trees" in my case) and then opened in PDF OCR X. Because I've put PDF -OCR X in non-interactive mode the files will automatically be OCR'ed without any -intervention from me. - -![Rule 3 Screenshot](/media/images{{ parent_url }}/rules-3-ocr.png "Rule 3") - -The fourth and final rule watches for the OCR'ed copies of our scans and runs -a script to move the originals to the trash once the searchable versions are ready. -It doesn't delete the files completely because I want a safety net in case something -goes wrong. - -![Rule 4 Screenshot](/media/images{{ parent_url }}/rules-4-clean.png "Rule 4") - -**Note:** make sure you change the Shell to `/usr/bin/python`. Here's the text of -the script so you can copy and paste it: - - import sys, os - - RM_CMD = r"""osascript -e 'tell app "Finder" to move the POSIX file "%s" to trash'""" - old_file = sys.argv[1].rsplit('.', 2)[0] - if os.path.exists(old_file): - os.system(RM_CMD % os.path.abspath(old_file)) - -Once these four rules are in place we can simply scan a document with Doxie or JotNot -and it will automatically be OCR'ed and placed in our "Dead Trees" folder, with no -intervention from us! - -[Hazel]: http://www.noodlesoft.com/hazel.php - -Backing Up ----------- - -A while ago I was using Mozy for full backups. Recently they changed their pricing so -it was no longer unlimited. When that happened I switched to [Backblaze][] and -couldn't be happier. - -Backblaze's UI is leaps and bounds above Mozy's, and they offer an option to generate -a secure encryption key for encrypting your backups. I highly recommend this, but be -sure to have a few copies of your key because you'll need it to restore your backups. - -Backblaze is also only $5 per month (less if you pay for a year in advance) for -unlimited backups which is definitely a bargain. As a bonus, they just released -a ["find my computer" feature][find-computer] that's kind of like a lightweight -version of [Undercover][], so it's an even better deal. - -[Backblaze]: http://www.backblaze.com/ -[Undercover]: http://www.orbicule.com/undercover/ -[find-computer]: http://blog.backblaze.com/2011/05/23/lost-your-computer-get-it-back-backblaze-launches-locate-my-computer/ - -Destroying the Originals ------------------------- - -Once the documents are scanned and backed up it's time to destroy the physical paper. -If you live in a rural area you could burn them for free. - -Those of us that can't start random fires need a paper shredder. I use a shredder -I picked up a long time ago -- any crosscut shredder will do the job. - -Summary -------- - -After all of this I've now got a mostly-automated system that lets me go paper-free. -The costs are: - -- Doxie Scanner: $160 -- JotNot: $7 -- PDF OCR X: $30 -- Hazel: $21 -- Backblaze: $5 per month - -For me the $218 initial cost is worth it. Now I can search all of my paper in -a instant and my apartment is much less cluttered. If you have the money to spare I'd -definitely consider trying it. - - {% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2011/05/paper-free.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2011/05/paper-free.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,230 @@ ++++ +title = "Going Paper-Free for $220" +snip = "It feels like the future!" +date = 2011-05-26T13:44:00Z +draft = false + ++++ + +It's 2011. Personal computers have been around and popular for well over a decade +now, and yet we still have to deal with a huge amount of physical paper. + +I've been wanting to go paper-free for a long time now. The advantages are obvious: + +- Paper takes up physical space in our homes that digital files don't. +- Digital files, if properly encrypted, are far more secure than sheets of paper that + could be stolen. +- Digital files can be searched in an instant, while papers have to be laboriously + sorted through. +- Digital files can be backed up perfectly and easily. + +After reading [this article][] I was psyched to scan and shred all the boxes of paper +sitting in my apartment, but the $420+ price tag was hard to swallow. I started +looking around for other options. + +[this article]: http://ryanwaggoner.com/2010/11/how-i-filled-two-dumpsters-and-went-paperless-with-the-fujitsu-scansnap-s1500/ + +Here are the requirements I have for any paper-free system: + +- The scanned files need to be OCR'ed so I can search them easily. I'm too lazy to + categorize and tag files manually. +- I need to be able to scan files anywhere. If I'm out at dinner I want to be able + to snap a picture of my receipt and tear it up right there. +- No "cloud" services allowed for unencrypted important documents. I simply don't + trust Google/Dropbox/etc enough to put my bank statements and such there. +- Files need to be backed up securely in case my apartment burns down. +- The entire process needs to be automated as much as possible, otherwise I'll get + lazy and not scan things. + +It's taken me a while, but I've finally got a system I'm happy with. This post will +describe each part and how they fit together. The total cost is about $220, $160 of +which is for a physical scanner. + +Note: I use OS X and an iPhone, so this post will focus on that platform. However, +the important pieces of software will run on Windows and I'm sure there are +Windows/Android equivalents to the other pieces. + +{{% toc %}} + +Scanning at Home +---------------- + +The first step to becoming paper-free is obviously scanning your documents. There +are a lot of scanners out there, some more expensive than others. I eventually +settled on a [Doxie][] for $160. + +[Doxie]: http://www.getdoxie.com/ + +I chose the Doxie because: + +- It's compact. +- It runs with a single USB cable. +- It's cross-platform. +- Its software has a great, polished UI. +- It has a "multiple-function button" that lets you control it without the + mouse/keyboard. + +The first, second, and last points mean that (with a USB extension cable) I can scan +documents while sitting on the couch and watching Netflix, which is critical for lazy +people like me. + +I set Doxie to save scans on my Desktop. The scanning process is pretty simple so +I won't describe it here. Check out Doxie's documentation for more information. + +**Note:** When I first received my Doxie and tried to calibrate it, it simply made +a grinding noise and wouldn't feed the paper. I emailed their tech support and +within half an hour I got a response back saying they were shipping me a replacement +immediately. + +When I got the replacement it worked like a charm. Their customer service was so +great that I'd still recommend the Doxie even though my first one was a dud. + +Scanning on the Go +------------------ + +As I mentioned before, I want to be able to scan things while out and about with my +iPhone. There are a bunch of iPhone document-scanning apps out there. I settled on +[JotNot][] for $7 because it has a decent UI and supports multiple-page PDFs. + +JotNot's UI is pretty easy to get the hang of so I won't go over it here. + +Once I finish scanning something I send the PDF to a [Dropbox][] folder called +"JotNot". + +I know I said in my requirements that "cloud" services weren't allowed, but I make an +exception for non-critical things that I'd be scanning with my phone. I don't care if +Dropbox knows how much I spent on dinner. + +[JotNot]: http://itunes.apple.com/us/app/jotnot-scanner-pro/id307868751?mt=8 +[Dropbox]: http://www.dropbox.com/ + +OCR'ing Scanned Documents +------------------------- + +The next step is to run the scanned PDFs through an OCR program so they can be +searched with Spotlight. + +I looked at a lot of OCR software and finally settled on [PDF OCR X][] for $30. It +has a simple interface, does a pretty good job at OCR'ing, has a free version so +I could try it out, and is cross-platform. + +Using it is simple: you drag a PDF onto the app and select your desired settings +(make sure to choose "searchable PDF" as the output format). The app will think for +a while and then create a new PDF next to the old one with the searchable text +embedded. + +Once you've done this once you should go into the preferences and change it to +non-interactive mode so that it won't prompt you for the settings every time you use +it. + +[PDF OCR X]: http://solutions.weblite.ca/pdfocrx/ + +Gluing Everything Together +-------------------------- + +So far we've got two folders with scanned PDFs and a method for OCR'ing them. The +next step is to automate the process. + +I use an app called [Hazel][] to do this. It's $21 for a license and well worth it. +We'll set up four rules to make our lives easier. + +Before we start we need to create two folders somewhere (you can name them whatever +you like): + +- Pending OCR: A folder to hold documents that are waiting to be OCR'ed. +- Dead Trees: A folder to hold the final, OCR'ed versions of our documents. + +The first rule watches the Desktop for scans from Doxie. Any files placed on the +Desktop whose name starts with "Doxie Doc" will be renamed to include the current +date and time, and then moved to the "Pending OCR" folder. + +![Rule 1 Screenshot](/media/images/blog/2011/05/rules-1-doxie.png "Rule 1") + +**Note:**: you'll need to click the `date created` bubble and then "Edit Date" to get +the time as well as the date into the filename. + +The second rule watches the "JotNot" folder for scans from the iPhone app. Any PDFs +that appear in here (i.e. that are synced down from Dropbox) will be moved to the +"Pending OCR" folder. We don't need to rename them like we did with the Doxie scans +because JotNot already includes the date and time of scans in the filenames by +default. + +![Rule 2 Screenshot](/media/images/blog/2011/05/rules-2-jotnot.png "Rule 2") + +Now that we've got all of our scans going into the same folder (with unique names) we +can set up a rule to OCR them. The third rule watches the "Pending OCR" folder for +PDFs. When a PDF lands in the folder it will be moved to its final destination +folder ("Dead Trees" in my case) and then opened in PDF OCR X. Because I've put PDF +OCR X in non-interactive mode the files will automatically be OCR'ed without any +intervention from me. + +![Rule 3 Screenshot](/media/images/blog/2011/05/rules-3-ocr.png "Rule 3") + +The fourth and final rule watches for the OCR'ed copies of our scans and runs +a script to move the originals to the trash once the searchable versions are ready. +It doesn't delete the files completely because I want a safety net in case something +goes wrong. + +![Rule 4 Screenshot](/media/images/blog/2011/05/rules-4-clean.png "Rule 4") + +**Note:** make sure you change the Shell to `/usr/bin/python`. Here's the text of +the script so you can copy and paste it: + + import sys, os + + RM_CMD = r"""osascript -e 'tell app "Finder" to move the POSIX file "%s" to trash'""" + old_file = sys.argv[1].rsplit('.', 2)[0] + if os.path.exists(old_file): + os.system(RM_CMD % os.path.abspath(old_file)) + +Once these four rules are in place we can simply scan a document with Doxie or JotNot +and it will automatically be OCR'ed and placed in our "Dead Trees" folder, with no +intervention from us! + +[Hazel]: http://www.noodlesoft.com/hazel.php + +Backing Up +---------- + +A while ago I was using Mozy for full backups. Recently they changed their pricing so +it was no longer unlimited. When that happened I switched to [Backblaze][] and +couldn't be happier. + +Backblaze's UI is leaps and bounds above Mozy's, and they offer an option to generate +a secure encryption key for encrypting your backups. I highly recommend this, but be +sure to have a few copies of your key because you'll need it to restore your backups. + +Backblaze is also only $5 per month (less if you pay for a year in advance) for +unlimited backups which is definitely a bargain. As a bonus, they just released +a ["find my computer" feature][find-computer] that's kind of like a lightweight +version of [Undercover][], so it's an even better deal. + +[Backblaze]: http://www.backblaze.com/ +[Undercover]: http://www.orbicule.com/undercover/ +[find-computer]: http://blog.backblaze.com/2011/05/23/lost-your-computer-get-it-back-backblaze-launches-locate-my-computer/ + +Destroying the Originals +------------------------ + +Once the documents are scanned and backed up it's time to destroy the physical paper. +If you live in a rural area you could burn them for free. + +Those of us that can't start random fires need a paper shredder. I use a shredder +I picked up a long time ago — any crosscut shredder will do the job. + +Summary +------- + +After all of this I've now got a mostly-automated system that lets me go paper-free. +The costs are: + +- Doxie Scanner: $160 +- JotNot: $7 +- PDF OCR X: $30 +- Hazel: $21 +- Backblaze: $5 per month + +For me the $218 initial cost is worth it. Now I can search all of my paper in +a instant and my apartment is much less cluttered. If you have the money to spare I'd +definitely consider trying it. + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2011/06/django-advice.html --- a/content/blog/2011/06/django-advice.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1064 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "Django Advice" - snip: "Some useful things I've learned." - created: 2011-06-30 08:30:00 - %} - - {% block article %} - -For the past year and a half or so I've been working full-time at [Dumbwaiter -Design][] doing [Django][] development. I've picked up a bunch of useful tricks along -the way that help me work, and I figured I'd share them. - -I'm sure there are better ways to do some of the things that I mention. If you know -of any feel free to hit me up on [Twitter][] and let me know. - -Also: this entry was written over several months, so if there are inconsistencies let -me know and I'll try to fix them. - -[Dumbwaiter Design]: http://dwaiter.com/ -[Django]: {{links.django}} -[Twitter]: http://twitter.com/stevelosh - -[TOC] - -Vagrant -------- - -I used to develop Django sites by running them on my OS X laptop locally and -deploying to a Linode VPS. I had a whole section of this post written up about -tricks and tips for working with that setup. - -Then I found [Vagrant][]. - -I just deleted the entire section of this post I wrote. - -Vagrant gives you a better way of working. You need to use it. - -[Vagrant]: http://vagrantup.com/ - -### Why Vagrant? - -If you haven't used it before, Vagrant is basically a tool for managing -[VirtualBox][] VMs. It makes it easy to start, pause, and resume VMs. Instead of -installing Django in a virtualenv and developing against that, you run a VM which -runs your site and develop against that. - -This may not sound like much, but it's kind of a big deal. The critical difference -is that you can now develop against the same setup that you'll be using in -production. - -This cuts out a huge amount of pain that stems from OS differences. Here are a few -examples off the top of my head: - -* URLField and MacPorts Python 2.5 on OS X. There's a [bug][] where using - verify\_exists will crash your site every time you save a model, unless you set - a particular environment variable with no debug information. Yeah, I spent - a couple of hours tracking that one down at work. Awesome. -* Installing PIL on OS X is no picnic. [homebrew][] makes things better, if you use it, - so this one isn't a huge deal. -* Every time you update Python in-place on your local machines, ALL of your - virtualenvs break because the Python binaries inside are linked against global - Python library files. Have fun recreating them. I hope you froze your - `requirements.txt` files before you updated. - -Using Vagrant and VMs means you can just worry about ONE operating system and its -quirks. It saves you a ton of time. - -Aside from that, there's another benefit to using Vagrant: it strongly encourages you -to learn and use an automated provisioning system. Support for Puppet and Chef is -built in. I chose Puppet, but if you prefer Chef that's cool too. - -You can also use other tools like Fabric or some simple scripts, but I'd strongly -recommend giving Puppet or Chef a fair shot. It's a lot to learn, but they're both -widely tested and very powerful. - -Because you're developing against a VM and deploying to a VM, you can reuse 90% of -the provisioning code across the two. - -When I make a new site, I do the following to initialize a new Vagrant VM: - -1. `vagrant up` (which runs Puppet to initialize the VM) -2. `fab dev bootstrap` - -When I'm ready to go live, I do the following: - -1. Buy a Linode VPS. -2. Run Puppet to initialize the VPS. -3. Enter the Linode info in my fabfile. -4. `fab prod bootstrap` - -No more screwing around with different paths, different versions of Nginx, different -versions of Python. When I'm developing something I can be pretty confident it will -"just work" in production without any major surprises. - -[VirtualBox]: http://www.virtualbox.org/ -[bug]: https://trac.macports.org/ticket/24421 -[homebrew]: http://mxcl.github.com/homebrew/ - -### Using Fabric to Stay Fast and Automate Everything - -One of the problems with this setup is that I can't just run `python manage.py -whatever` any more because I need it to run on the VM. - -To get around this I've created many simple [Fabric][] tasks to automate the common -things I need to do. Fabric is an awesome little Python utility for scripting tasks -(like deployments). We use it constantly at Dumbwaiter. Here are a few examples -from our fabfiles. - -[Fabric]: http://fabfile.org/ - -This first set is for running abitrary commands easily. - -`cmd` and `vcmd` will `cd` into the site directory on the VM and run a command of my -choosing. `vcmd` will prefix the command with the path to the virtualenv's `bin` -directory, so I can do something like `fab dev vcmd`, `pip install markdown`. - -The `sdo` commands do the same thing, but `sudo`'ed. - - :::python - def cmd(cmd=""): - '''Run a command in the site directory. Usable from other commands or the CLI.''' - require('site_path') - - - if not cmd: - sys.stdout.write(_cyan("Command to run: ")) - cmd = raw_input().strip() - - if cmd: - with cd(env.site_path): - run(cmd) - - def sdo(cmd=""): - '''Sudo a command in the site directory. Usable from other commands or the CLI.''' - require('site_path') - - if not cmd: - sys.stdout.write(_cyan("Command to run: sudo ")) - cmd = raw_input().strip() - - if cmd: - with cd(env.site_path): - sudo(cmd) - - def vcmd(cmd=""): - '''Run a virtualenv-based command in the site directory. Usable from other commands or the CLI.''' - require('site_path') - require('venv_path') - - if not cmd: - sys.stdout.write(_cyan("Command to run: %s/bin/" % env.venv_path.rstrip('/'))) - cmd = raw_input().strip() - - if cmd: - with cd(env.site_path): - run(env.venv_path.rstrip('/') + '/bin/' + cmd) - - def vsdo(cmd=""): - '''Sudo a virtualenv-based command in the site directory. Usable from other commands or the CLI.''' - require('site_path') - require('venv_path') - - if not cmd: - sys.stdout.write(_cyan("Command to run: sudo %s/bin/" % env.venv_path.rstrip('/'))) - cmd = raw_input().strip() - - if cmd: - with cd(env.site_path): - sudo(env.venv_path.rstrip('/') + '/bin/' + cmd) - -This next set is just some common commands that I need to run often. - - :::python - def syncdb(): - '''Run syncdb.''' - require('site_path') - require('venv_path') - - with cd(env.site_path): - run(_python('manage.py syncdb --noinput')) - - def collectstatic(): - '''Collect static media.''' - require('site_path') - require('venv_path') - - with cd(env.site_path): - sudo(_python('manage.py collectstatic --noinput')) - - def rebuild_index(): - '''Rebuild the search index.''' - require('site_path') - require('venv_path') - require('process_owner') - - with cd(env.site_path): - sudo(_python('manage.py rebuild_index')) - sudo('chown -R %s .xapian' % env.process_owner) - - def update_index(): - '''Update the search index.''' - require('site_path') - require('venv_path') - require('process_owner') - - with cd(env.site_path): - sudo(_python('manage.py update_index')) - sudo('chown -R %s .xapian' % env.process_owner) - -We also use Fabric to automate some of the more complex things we need to do. - -This task `curl`'s the site's home page to make sure we haven't completely borked -things. We use it in lots of other tasks as a sanity check. - - :::python - def check(): - '''Check that the home page of the site returns an HTTP 200.''' - require('site_url') - - print('Checking site status...') - - if not '200 OK' in local('curl --silent -I "%s"' % env.site_url, capture=True): - _sad() - else: - _happy() - -The `_happy` and `_sad` functions just print out some simple messages to get our -attention: - - :::python - from fabric.colors import red, green - - def _happy(): - print(green('\nLooks good from here!\n')) - - def _sad(): - print(red(r''' - ___ ___ - / /\ /__/\ - / /::\ \ \:\ - / /:/\:\ \__\:\ - / /:/ \:\ ___ / /::\ - /__/:/ \__\:\ /__/\ /:/\:\ - \ \:\ / /:/ \ \:\/:/__\/ - \ \:\ /:/ \ \::/ - \ \:\/:/ \ \:\ - \ \::/ \ \:\ - \__\/ \__\/ - ___ ___ ___ ___ - /__/\ / /\ / /\ / /\ ___ - \ \:\ / /::\ / /:/_ / /:/_ /__/\ - \ \:\ / /:/\:\ / /:/ /\ / /:/ /\ \ \:\ - _____\__\:\ / /:/ \:\ / /:/ /:/_ / /:/ /::\ \ \:\ - /__/::::::::\ /__/:/ \__\:\ /__/:/ /:/ /\ /__/:/ /:/\:\ \ \:\ - \ \:\~~\~~\/ \ \:\ / /:/ \ \:\/:/ /:/ \ \:\/:/~/:/ \ \:\ - \ \:\ ~~~ \ \:\ /:/ \ \::/ /:/ \ \::/ /:/ \__\/ - \ \:\ \ \:\/:/ \ \:\/:/ \__\/ /:/ __ - \ \:\ \ \::/ \ \::/ /__/:/ /__/\ - \__\/ \__\/ \__\/ \__\/ \__\/ - - - Something seems to have gone wrong! - You should probably take a look at that. - ''')) - -This one is for when `python manage.py reset APP` is broken because you've changed -some `db_column` names and Django chokes because of some constraits and you just want -to **reset the fucking app**. - -It's the "NUKE IT FROM ORBIT!!" option. - - :::python - def KILL_IT_WITH_FIRE(app): - require('site_path') - require('venv_path') - - with cd(env.site_path): - # Generate and download the reset SQL. - sudo(_python('manage.py sqlreset %s > reset.orig.sql' % app)) - get('reset.orig.sql') - - with open('reset.sql', 'w') as f: - with open('reset.orig.sql') as orig: - # Step through the first chunk of the file (the "drop" part). - line = orig.readline() - while not line.startswith('CREATE'): - if 'CONSTRAINT' in line: - # Don't write out CONSTRAINT lines. - # They're a problem when you change db_colum names. - pass - elif 'DROP TABLE' in line: - # Cascade drops. - # Hence with "with fire" part of this task's name. - line = line[:-2] + ' CASCADE;\n' - f.write(line) - else: - # Write other lines through untoched. - f.write(line) - line = orig.readline() - - # Write out the rest of the file untouched. - f.write(line) - f.write(orig.read()) - - # Upload the processed SQL file. - put('reset.sql', os.path.join(env.site_path, 'reset.ready.sql'), use_sudo=True) - - with cd(env.site_path): - # Use the SQL to reset the app, and fake a migration. - run(_python('manage.py dbshell < reset.ready.sql')) - sudo(_python('manage.py migrate --fake --delete-ghost-migrations ' + app)) - -This task uses Mercurial's local tags to add a `production` or `staging` tag in your local -repository, so you can easy see where the production/staging servers are at -compared to your local repo. - - :::python - def retag(): - '''Check which revision the site is at and update the local tag. - - Useful if someone else has deployed (which makes your production/staging local - tag incorrect. - ''' - require('site_path', provided_by=['prod', 'stag']) - require('env_name', provided_by=['prod', 'stag']) - - with cd(env.site_path): - current = run('hg id --rev . --quiet').strip(' \n+') - - local('hg tag --local --force %s --rev %s' % (env.env_name, current)) - -This task tails the Gunicorn logs on the server so you can quickly find out what's -happening when things blow up. - - :::python - def tailgun(follow=''): - """Tail the Gunicorn log file.""" - require('site_path') - - with cd(env.site_path): - if follow: - run('tail -f .gunicorn.log') - else: - run('tail .gunicorn.log') - -We've got a lot of other tasks but they're pretty specific to our setup. - -Wrangling Databases with South ------------------------------- - -If you're not using [South][], you need to start. Now. - -No, really, I'll wait. Take 30 minutes, try the [tutorial][Southtut], wrap your head -around it and come back. It's far more important than this blog post. - -[South]: http://south.aeracode.org/ -[Southtut]: http://south.aeracode.org/docs/tutorial/index.html - -### Useful Fabric Tasks - -South is awesome but its commands are very long-winded. Here's the set of fabric -tasks I use to save quite a bit of typing: - - :::python - def migrate(args=''): - '''Run any needed migrations.''' - require('site_path') - require('venv_path') - - with cd(env.site_path): - sudo(_python('manage.py migrate ' + args)) - - def migrate_fake(args=''): - '''Run any needed migrations with --fake.''' - require('site_path') - require('venv_path') - - with cd(env.site_path): - sudo(_python('manage.py migrate --fake ' + args)) - - def migrate_reset(args=''): - '''Run any needed migrations with --fake. No, seriously.''' - require('site_path') - require('venv_path') - - with cd(env.site_path): - sudo(_python('manage.py migrate --fake --delete-ghost-migrations ' + args)) - -Remember that running a migration without specifying an app will migrate everything, -so a simple `fab dev migrate` will do the trick. - -Watching for Changes ---------------------- - -When developing locally you'll want to make a change to your code and have the server -reload that code automatically. The Django development server does this, and we can -hack it into our Vagrant/Gunicorn setup too. - -First, add a `monitor.py` file at the root of your project (I believe I found this -code [here][monitor], but I may be wrong): - - :::python - import os - import sys - import time - import signal - import threading - import atexit - import Queue - - _interval = 1.0 - _times = {} - _files = [] - - _running = False - _queue = Queue.Queue() - _lock = threading.Lock() - - def _restart(path): - _queue.put(True) - prefix = 'monitor (pid=%d):' % os.getpid() - print >> sys.stderr, '%s Change detected to \'%s\'.' % (prefix, path) - print >> sys.stderr, '%s Triggering process restart.' % prefix - os.kill(os.getpid(), signal.SIGINT) - - def _modified(path): - try: - # If path doesn't denote a file and were previously - # tracking it, then it has been removed or the file type - # has changed so force a restart. If not previously - # tracking the file then we can ignore it as probably - # pseudo reference such as when file extracted from a - # collection of modules contained in a zip file. - - if not os.path.isfile(path): - return path in _times - - # Check for when file last modified. - - mtime = os.stat(path).st_mtime - if path not in _times: - _times[path] = mtime - - # Force restart when modification time has changed, even - # if time now older, as that could indicate older file - # has been restored. - - if mtime != _times[path]: - return True - except: - # If any exception occured, likely that file has been - # been removed just before stat(), so force a restart. - - return True - - return False - - def _monitor(): - while 1: - # Check modification times on all files in sys.modules. - - for module in sys.modules.values(): - if not hasattr(module, '__file__'): - continue - path = getattr(module, '__file__') - if not path: - continue - if os.path.splitext(path)[1] in ['.pyc', '.pyo', '.pyd']: - path = path[:-1] - if _modified(path): - return _restart(path) - - # Check modification times on files which have - # specifically been registered for monitoring. - - for path in _files: - if _modified(path): - return _restart(path) - - # Go to sleep for specified interval. - - try: - return _queue.get(timeout=_interval) - except: - pass - - _thread = threading.Thread(target=_monitor) - _thread.setDaemon(True) - - def _exiting(): - try: - _queue.put(True) - except: - pass - _thread.join() - - atexit.register(_exiting) - - def track(path): - if not path in _files: - _files.append(path) - - def start(interval=1.0): - global _interval - if interval < _interval: - _interval = interval - - global _running - _lock.acquire() - if not _running: - prefix = 'monitor (pid=%d):' % os.getpid() - print >> sys.stderr, '%s Starting change monitor.' % prefix - _running = True - _thread.start() - _lock.release() - -Next add a `post_fork` hook to your Gunicorn config file that uses the monitor to -watch for changes: - - :::python - def post_fork(server, worker): - import monitor - import local_settings - if local_settings.DEBUG: - server.log.info("Starting change monitor.") - monitor.start(interval=1.0) - -Now the Gunicorn server will automatically restart whenever code is changed. Use -whatever method for determining debug status that you like. We use -`local_settings.py` files which all have `DEBUG` variables, so that works for us. - -It will *not* restart when you add new code (e.g. when you install a new app), so -you'll need to handle that manually with `fab dev restart`, but that's not too bad! - -[monitor]: http://code.google.com/p/modwsgi/wiki/ReloadingSourceCode - -### Using the Werkzeug Debugger with Gunicorn - -The final piece of the puzzle is being able to use the fantastic [Werkzeug -Debugger][debug] while running on the development VM with Gunicorn. - -To do this, create a `debug_wsgi.py` file at the root of your project: - - :::python - import os - import sys - import site - - parent = os.path.dirname - site_dir = parent(os.path.abspath(__file__)) - project_dir = parent(parent(os.path.abspath(__file__))) - - sys.path.insert(0, project_dir) - sys.path.insert(0, site_dir) - - site.addsitedir('VIRTUALENV_SITE_PACKAGES') - - from django.core.management import setup_environ - import settings - setup_environ(settings) - - import django.core.handlers.wsgi - application = django.core.handlers.wsgi.WSGIHandler() - - from werkzeug.debug import DebuggedApplication - application = DebuggedApplication(application, evalex=True) - - def null_technical_500_response(request, exc_type, exc_value, tb): - raise exc_type, exc_value, tb - from django.views import debug - debug.technical_500_response = null_technical_500_response - -Have Gunicorn use this file to run your development server with `gunicorn -debug_wsgi:application`. - -Make sure to replace `'VIRTUALENV_SITE_PACKAGES'` with the _full_ path to your -virtualenv's `site_packages` directory. You might want to make this a setting in -a machine-specific settings file. - -[debug]: http://werkzeug.pocoo.org/docs/debug/ - -### Pulling Uploads - -Once you give a client access to a site they'll probably be uploading images (through -Django's built-in file uploading features or with [django-filebrowser][]). - -When you're making changes locally it's often useful to have these uploaded files on -your local VM, otherwise you end up with a bunch of broken images. - -Here's a simple Fabric task that will pull down all the uploads from the server: - - :::python - def pull_uploads(): - '''Copy the uploads from the site to your local machine.''' - require('uploads_path') - - sudo('chmod -R a+r "%s"' % env.uploads_path) - - rsync_command = r"""rsync -av -e 'ssh -p %s' %s@%s:%s %s""" % ( - env.port, - env.user, env.host, - env.uploads_path.rstrip('/') + '/', - 'media/uploads' - ) - print local(rsync_command, capture=False) - -You might be wondering about the line that strips `/` characters and then adds them -back in. `rsync` does different things depending on whether you end a path with -a `/`, so this is actually pretty important. - -In your host task you'll need to set the `uploads_path` variable to something like -this: - - :::python - import os - env.site_path = os.path.join('var', 'www', 'myproject') - env.uploads_path = os.path.join(env.site_path, 'media', 'uploads') - -Now you can run `fab production pull_uploads` to pull down all the files people have -uploaded to the production server. - -[django-filebrowser]: http://code.google.com/p/django-filebrowser/ - -### Preventing Accidents - -Deploying to test and staging servers should be quick and easy. Deploying to -production servers should be harder to prevent people from accidentally doing it. - -I've created a little function that I call before deploying to production servers. -It forces me to type in random words from the system word list before proceeding to -make sure I *really* know what I'm doing: - - :::python - import os, random - - from fabric.api import * - from fabric.operations import prompt - from fabric.utils import abort - - WORDLIST_PATHS = [os.path.join('/', 'usr', 'share', 'dict', 'words')] - DEFAULT_MESSAGE = "Are you sure you want to do this?" - WORD_PROMPT = ' [%d/%d] Type "%s" to continue (^C quits): ' - - def prevent_horrible_accidents(msg=DEFAULT_MESSAGE, horror_rating=1): - """Prompt the user to enter random words to prevent doing something stupid.""" - - valid_wordlist_paths = [wp for wp in WORDLIST_PATHS if os.path.exists(wp)] - - if not valid_wordlist_paths: - abort('No wordlists found!') - - with open(valid_wordlist_paths[0]) as wordlist_file: - words = wordlist_file.readlines() - - print msg - - for i in range(int(horror_rating)): - word = words[random.randint(0, len(words))].strip() - p_msg = WORD_PROMPT % (i+1, horror_rating, word) - answer = prompt(p_msg, validate=r'^%s$' % word) - -You may need to adjust `WORDLIST_PATHS` if you're not on OS X. - -Working with Third-Party Apps ------------------------------ - -One of the best parts about working with Django is that many problems have already -been solved and the solutions have been released as open-source applications. - -We use quite a few open-source apps, and there are a couple of tricks I've learned to -make working with them easier. - -### Installing Apps from Repositories - -If I'm going to use an open-source Django app in a project I'll almost always install -it as an editable repository on the VM with `pip install -e`. - -Others may disagree with me on this, but I think it's the best way to work. - -Often I'll find a bug that I think may be in one of the third-party apps I'm using. -Installing the apps as repositories makes it easy to read their source and figure out -if the bug is really in the app. - -If the bug *is* in the third-party app having the app installed as a repository makes -it simple to fix the bug, fork the project on BitBucket or GitHub, send a pull -request, and get back to work. - -### Mirroring Repositories - -One problem we've run into at Dumbwaiter is that the repos for third-party apps we -use are scattered across GitHub, BitBucket, Google Code, and other servers. If any -one of these services goes down we're stuck waiting for it to come back up. - -A while ago I took half a day and consolidated all of these repos onto one of the -servers that we control. The basic process went like this: - -* Use [hg-git][] and [hgsubversion][] to convert the git and SVN repos to Mercurial - repos. -* Set up a master `mirror` Mercurial repo with all the app repos as subrepos. -* Push the master repo and all the subrepos up to one of our Linodes. - -Now we can use `-e ssh://hg@OUR_LINODE/mirror/APP@REV_THAT_WORKS#egg=APP` in our -`requirements.txt` files to install apps from our mirror. When we want to update our -dependencies we can simply pull from the upstream repos and commit in the mirror -repo. - -If our mirror goes down it's not a big deal, because we have far bigger problems to -worry about than new projects. - -I wrote a few scripts to automate updating apps and such, but they're extremely hacky -so I don't want to post them here. Take half a day and write your own set -- it's -definitely worth it to have your own mirror of your specific dependencies. - -[hg-git]: http://hg-git.github.com/ -[hgsubversion]: https://bitbucket.org/durin42/hgsubversion/wiki/Home - -### Using BCVI to Edit Files - -I said that when I find a bug that I think is in a third-party app I'll poke around -with the app and try to figure it out. But since all the apps are installed in -a virtualenv on the Vagrant VM it might seem like it's a pain in the ass to edit -those files! - -Luckily [BCVI][] exists. It's a utility that opens a "back channel" to your local -machine when you SSH and lets you run `vi FILE` to open that file in -Vim/MacVim/GVim/etc on your *local* machine. When you save the file it uploads it -back to the server automatically for you. - -It can be a bit tricky to set up, but it's worth it. Trust me. - -[BCVI]: http://sshmenu.sourceforge.net/articles/bcvi/ - -Improving the Admin Interface ------------------------------ - -I'm going to be honest: Django's admin interface is the main reason I'm still using -it. Other frameworks like [Flask][] are great, but Django's admin saves me -*ridiculous* amounts of time when I'm making simple CRUD sites for clients. - -That said, the Django admin isn't the prettiest thing around, but we can give it -a facelift. - -[Flask]: http://flask.pocoo.org/ - -### Enter Grappelli - -[Grappelli][] is a Django app that reskins the admin interface beautifully. It also -adds some functionality like drag-and-drop reordering of inlines, and allows you to -customize the dashboard to your liking. *Every* Django site I work on uses Grappelli --- it's just that good. - -The downside of Grappelli is that it changes quite a lot and breaks backwards -compatibility at the drop of a hat. - -If you're going to use Grappelli you *must* freeze your requirements.txt files and -work with a single version at a time. Trying to always work from the trunk will make -you drink. - -[Grappelli]: http://django-grappelli.readthedocs.org/ - -### An Ugly Hack to Show Usable Foreign Key Fields - -A limitation of both Grappelli and the stock Django admin is that it seems like you -can't easily show fields from related models in the admin list view. - -For example, if you're new to Django you might expect this to work: - - :::python - class BlogEntryAdmin(admin.ModelAdmin): - list_display = ('title', 'author__name') - -Unfortunately Django chokes on the `author__name` lookup. You can *display* the name -without too much fuss: - - :::python - class BlogEntryAdmin(admin.ModelAdmin): - list_display = ('title', 'author_name') - - def author_name(self, obj): - return obj.name - -That will display the name just fine. However, it won't be a fully-fledged column in -the Django admin because you can't sort on it. - -It may seem like this is the end -- if it could be a fully-functional field, why -wouldn't Django just let you use `author__name`? Luckily we can add one more line to -fix the problem: - - :::python - class BlogEntryAdmin(admin.ModelAdmin): - list_display = ('title', 'author_name') - - def author_name(self, obj): - return obj.name - author_name.admin_order_field = 'author__name' - -Now the author name has all the functionality of a real `list_display` entry. - -Using Django-Annoying ---------------------- - -If you haven't heard of [django-annoying][] you should definitely check it out. It's -got a bunch of miscellaneous functions that fix some common, annoying parts of -Django. - -My two personal favorites from the package are a pair of decorators that help make -your views much, much cleaner. - -[django-annoying]: https://bitbucket.org/offline/django-annoying/wiki/Home - -### The render\_to Decorator - -The decorator is called `render_to` and it eliminates the ugly `render_to_response` -calls that Django normally forces you to use in every single view. - -Normally you'd use something like this: - - :::python - def videos(request): - videos = Video.objects.all() - return render_to_response('video_list.html', { 'videos': videos }, - context_instance=RequestContext(request)) - -With `render_to` your view gets much cleaner: - - :::python - @render_to('video_list.html') - def videos(request): - videos = Video.objects.all() - return { 'videos': videos } - -Less typing `context_instance=...` over and over, and less syntax to remember. - -Yes, I know about Django 1.3's `render` shortcut. You have to type `request` every -single time with `render`, so the `render_to` decorator still wins. - -### The ajax\_request Decorator - -The `ajax_request` decorator is like `render_to` for AJAX requests. You simply -return a Python dictionary from your view and the decorator handles the JSON encoding -and such: - - :::python - @ajax_request - def ajax_get_entries(request): - blog_entries = BlogEntry.objects.all() - return { 'entries': [(entry.title, entry.get_absolute_url()) - for entry in entries]} - -Templating Tricks ------------------ - -I'm not a frontend developer, but I've done my share of HTML hacking at Dumbwaiter. -Here are a few of the tricks I've learned. - -### Null Checks and Fallbacks - -A common pattern I see in Django templates looks like this: - - :::django - {% templatetag openblock %} if business.title {% templatetag closeblock %} - {% templatetag openvariable %} business.title {% templatetag closevariable %} - {% templatetag openblock %} else {% templatetag closeblock %} - {% templatetag openvariable %} business.short_title {% templatetag closevariable %} - {% templatetag openblock %} endif {% templatetag closeblock %} - -Here's a simpler way to do that: - - :::django - {% templatetag openblock %} firstof business.title business.short_title {% templatetag closeblock %} - -`firstof` will return the first non-Falsy item in its arguments. - -### Manipulating Query Strings - -Query strings are normally not a big deal, but every once in a while you'll have -a model listing page where you need to filter by category, and number of spaces, and -tags, etc all at once. - -If you're trying to manage GET queries manually it can get pretty hairy very fast. - -[This Django snippet][qstring] makes working with query strings in templates -a breeze. - -[qstring]: http://djangosnippets.org/snippets/2237/ - -### Satisfying Your Designer with Typogrify - -If you haven't heard of [Typogrify][] you should take a look at it. It makes it easy -to add all the typographic goodness your designers are looking for. - -[Typogrify]: http://code.google.com/p/typogrify/ - -The Flat Page Trainwreck ------------------------- - -Creating a site for a client is very different than creating a site for yourself. -For pretty much every client we've dealt with we've heard: "can't we just create -a new page at /drink-special/ for this special deal we're running?" - -Having clients go through you to make new pages is simply too much overhead. We -needed a way to let clients create new pages (like `/drink-special/`) on the fly, -without our intervention. - -Django has a "flatpages" app that solves this problem. Kind of. - -When using flat pages clients need to do two things that are often too much for -non-technical people: - -* Manage URLs manually. -* Write all content as raw HTML in a single text field. - -We've tried a lot of Django CMS apps at Dumbwaiter, and none of them made us happy. -They all seemed to have some or all of the following problems: - -* They take over your site and make you write a "Django-WhateverCMS site" instead of - a "Django site". -* They're extremely feature-rich and complicated with features like - internationalization, redirects, versions, and many others. This is great if you - need the flexibility, but bad if your clients just need to create a couple of - pages. -* They break `APPEND_TRAILING_SLASH` and make you clutter your `urls.py` files with - a bunch of extra code ot handle this. - -I finally got fed up and wrote my own Django CMS app: [Stoat][]. Stoat is designed -to be sleek, with only the features that our clients need. - -It's not officially version 1.0 yet, but we're using it for a few clients and it's -working well. Check it out if you're looking for a more lightweight CMS app. - -[Stoat]: http://stoat.rtfd.org/ - -Editing with Vim ----------------- - -I [use Vim][vimpost] to edit everything. Naturally I've found a bunch of plugins, -mappings and other tricks that make it even better when working on Django projects. - -[vimpost]: /blog/2010/09/coming-home-to-vim/ - -### Vim for Django - -There are a lot of ways to make Vim work with Django. I won't go into all of them in -this post, but a good place to start is [this Django wiki page][vimdjango]. - -[vimdjango]: https://code.djangoproject.com/wiki/UsingVimWithDjango - -### Filetype Mappings - -Most files in a Django project have one of two extensions: `.py` and `.html`. -Unfortunately these extensions aren't unique to Django, so Vim doesn't automatically -set the correct `filetype` when you open one. - -I've added a few mappings to my `.vimrc` to make it quick and easy to set the correct -`filetype`: - - :::vim - nnoremap _dt :set ft=htmldjango - nnoremap _pd :set ft=python.django - -I also have a few autocommands that set the filetype for me when I'm editing a file -whose name "sounds like" a Django file: - - :::vim - au BufNewFile,BufRead admin.py setlocal filetype=python.django - au BufNewFile,BufRead urls.py setlocal filetype=python.django - au BufNewFile,BufRead models.py setlocal filetype=python.django - au BufNewFile,BufRead views.py setlocal filetype=python.django - au BufNewFile,BufRead settings.py setlocal filetype=python.django - au BufNewFile,BufRead forms.py setlocal filetype=python.django - -### Python Sanity Checking - -Lets be honest here: it takes a lot of work to turn Vim into an "IDE", and even then -it doesn't reach the level of something like Eclipse for Java. Anyone who claims it -has the same levels of integration and functionality is simply lying. - -With that said I'll make an opinionated statement that is going to piss some of you -off. - -**I am a programmer, not an IDE operator.** - -I know Python. - -I know Django. - -I don't need to hit Cmd+Space twice for every line of code I write. - -When someone asks me "how do you run your site" I do **not** answer: "click the green -triangle in Eclipse". - -However, I am human. I do stupid things like forgetting a colon or forgetting an -import. To help me with those problems I've turned to [Syntastic][] and Kevin -Watters' [Pyflakes fork][] for Vim. - -Syntastic is a Vim plugin that adds on-the-fly syntax-checking for many different -file formats. If you have Pyflakes installed it will automatically show you errors -in your code. - -Pyflakes doesn't have IDE-level integration with your code. It doesn't check that -whatever libraries you `import` actually exist. It simply checks that your files are -probably-valid Python, and tells you when they're not. - -This is enough for me. It catches the stupid mistakes I make. The less-stupid, -more-subtle mistakes slip by it, but to be fair many of them would have slipped by an -"IDE" as well. - -[Pyflakes fork]: https://github.com/kevinw/pyflakes -[Syntastic]: http://www.vim.org/scripts/script.php?script_id=2736 - -### Javascript Sanity Checking and Folding - -Syntastic also supports Javascript if you have Javascript Lint installed (`brew -install jsl` on OS X). It's not perfect but it *will* catch things like using -trailing commas in object literals. - -Some people like using CTags to get an overview of their code. I take a more -low-tech approach and am in love with code folding. When I fold my code -I automatically get an overview of everything in each file. - -By default Vim doesn't fold Javascript files, but you can add some basic, perfectly -serviceable folding with these two lines in your .vimrc: - - :::vim - au FileType javascript setlocal foldmethod=marker - au FileType javascript setlocal foldmarker={,} - - -### Django Autocommands - -I *rarely* work with raw HTML files any more. Whenever I open a file ending in -`.html` it's almost always a Django template (or a [Jinja][] template, which has -a very similar syntax). I've added an autocommand to automatically set the correct -filetype whenever I open a `.html` file: - -[Jinja]: http://jinja.pocoo.org/ - - :::vim - au BufNewFile,BufRead *.html setlocal filetype=htmldjango - -I also have some autocommands that tweak how a few specific files are handled: - - :::vim - au BufNewFile,BufRead urls.py setlocal nowrap - au BufNewFile,BufRead settings.py normal! zR - au BufNewFile,BufRead dashboard.py normal! zR - -This automatically unfolds `urls.py`, `dashboard.py` and `settings.py` (I prefer -seeing those unfolded) and unsets line wrapping for `urls.py` (lines in a `urls.py` -file can get long and are hard to read when wrapped). - -Conclusion ----------- - -I hope that this longer-than-expected blog entry has given you at least one or two -things to think about. - -I've learned a lot while working with Django for Dumbwaiter every day, but I'm sure -there's still a lot I've missed. If you see something I could be doing better please -let me know! - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2011/06/django-advice.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2011/06/django-advice.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,1088 @@ ++++ +title = "Django Advice" +snip = "Some useful things I've learned." +date = 2011-06-30T08:30:00Z +draft = false + ++++ + +For the past year and a half or so I've been working full-time at [Dumbwaiter +Design][] doing [Django][] development. I've picked up a bunch of useful tricks along +the way that help me work, and I figured I'd share them. + +I'm sure there are better ways to do some of the things that I mention. If you know +of any feel free to hit me up on [Twitter][] and let me know. + +Also: this entry was written over several months, so if there are inconsistencies let +me know and I'll try to fix them. + +[Dumbwaiter Design]: http://dwaiter.com/ +[Django]: {{links.django}} +[Twitter]: http://twitter.com/stevelosh + +{{% toc %}} + +Vagrant +------- + +I used to develop Django sites by running them on my OS X laptop locally and +deploying to a Linode VPS. I had a whole section of this post written up about +tricks and tips for working with that setup. + +Then I found [Vagrant][]. + +I just deleted the entire section of this post I wrote. + +Vagrant gives you a better way of working. You need to use it. + +[Vagrant]: http://vagrantup.com/ + +### Why Vagrant? + +If you haven't used it before, Vagrant is basically a tool for managing +[VirtualBox][] VMs. It makes it easy to start, pause, and resume VMs. Instead of +installing Django in a virtualenv and developing against that, you run a VM which +runs your site and develop against that. + +This may not sound like much, but it's kind of a big deal. The critical difference +is that you can now develop against the same setup that you'll be using in +production. + +This cuts out a huge amount of pain that stems from OS differences. Here are a few +examples off the top of my head: + +* URLField and MacPorts Python 2.5 on OS X. There's a [bug][] where using + verify\_exists will crash your site every time you save a model, unless you set + a particular environment variable with no debug information. Yeah, I spent + a couple of hours tracking that one down at work. Awesome. +* Installing PIL on OS X is no picnic. [homebrew][] makes things better, if you use it, + so this one isn't a huge deal. +* Every time you update Python in-place on your local machines, ALL of your + virtualenvs break because the Python binaries inside are linked against global + Python library files. Have fun recreating them. I hope you froze your + `requirements.txt` files before you updated. + +Using Vagrant and VMs means you can just worry about ONE operating system and its +quirks. It saves you a ton of time. + +Aside from that, there's another benefit to using Vagrant: it strongly encourages you +to learn and use an automated provisioning system. Support for Puppet and Chef is +built in. I chose Puppet, but if you prefer Chef that's cool too. + +You can also use other tools like Fabric or some simple scripts, but I'd strongly +recommend giving Puppet or Chef a fair shot. It's a lot to learn, but they're both +widely tested and very powerful. + +Because you're developing against a VM and deploying to a VM, you can reuse 90% of +the provisioning code across the two. + +When I make a new site, I do the following to initialize a new Vagrant VM: + +1. `vagrant up` (which runs Puppet to initialize the VM) +2. `fab dev bootstrap` + +When I'm ready to go live, I do the following: + +1. Buy a Linode VPS. +2. Run Puppet to initialize the VPS. +3. Enter the Linode info in my fabfile. +4. `fab prod bootstrap` + +No more screwing around with different paths, different versions of Nginx, different +versions of Python. When I'm developing something I can be pretty confident it will +"just work" in production without any major surprises. + +[VirtualBox]: http://www.virtualbox.org/ +[bug]: https://trac.macports.org/ticket/24421 +[homebrew]: http://mxcl.github.com/homebrew/ + +### Using Fabric to Stay Fast and Automate Everything + +One of the problems with this setup is that I can't just run `python manage.py +whatever` any more because I need it to run on the VM. + +To get around this I've created many simple [Fabric][] tasks to automate the common +things I need to do. Fabric is an awesome little Python utility for scripting tasks +(like deployments). We use it constantly at Dumbwaiter. Here are a few examples +from our fabfiles. + +[Fabric]: http://fabfile.org/ + +This first set is for running abitrary commands easily. + +`cmd` and `vcmd` will `cd` into the site directory on the VM and run a command of my +choosing. `vcmd` will prefix the command with the path to the virtualenv's `bin` +directory, so I can do something like `fab dev vcmd`, `pip install markdown`. + +The `sdo` commands do the same thing, but `sudo`'ed. + +```python +def cmd(cmd=""): + '''Run a command in the site directory. Usable from other commands or the CLI.''' + require('site_path') + + + if not cmd: + sys.stdout.write(_cyan("Command to run: ")) + cmd = raw_input().strip() + + if cmd: + with cd(env.site_path): + run(cmd) + +def sdo(cmd=""): + '''Sudo a command in the site directory. Usable from other commands or the CLI.''' + require('site_path') + + if not cmd: + sys.stdout.write(_cyan("Command to run: sudo ")) + cmd = raw_input().strip() + + if cmd: + with cd(env.site_path): + sudo(cmd) + +def vcmd(cmd=""): + '''Run a virtualenv-based command in the site directory. Usable from other commands or the CLI.''' + require('site_path') + require('venv_path') + + if not cmd: + sys.stdout.write(_cyan("Command to run: %s/bin/" % env.venv_path.rstrip('/'))) + cmd = raw_input().strip() + + if cmd: + with cd(env.site_path): + run(env.venv_path.rstrip('/') + '/bin/' + cmd) + +def vsdo(cmd=""): + '''Sudo a virtualenv-based command in the site directory. Usable from other commands or the CLI.''' + require('site_path') + require('venv_path') + + if not cmd: + sys.stdout.write(_cyan("Command to run: sudo %s/bin/" % env.venv_path.rstrip('/'))) + cmd = raw_input().strip() + + if cmd: + with cd(env.site_path): + sudo(env.venv_path.rstrip('/') + '/bin/' + cmd) +``` + +This next set is just some common commands that I need to run often. + +```python +def syncdb(): + '''Run syncdb.''' + require('site_path') + require('venv_path') + + with cd(env.site_path): + run(_python('manage.py syncdb --noinput')) + +def collectstatic(): + '''Collect static media.''' + require('site_path') + require('venv_path') + + with cd(env.site_path): + sudo(_python('manage.py collectstatic --noinput')) + +def rebuild_index(): + '''Rebuild the search index.''' + require('site_path') + require('venv_path') + require('process_owner') + + with cd(env.site_path): + sudo(_python('manage.py rebuild_index')) + sudo('chown -R %s .xapian' % env.process_owner) + +def update_index(): + '''Update the search index.''' + require('site_path') + require('venv_path') + require('process_owner') + + with cd(env.site_path): + sudo(_python('manage.py update_index')) + sudo('chown -R %s .xapian' % env.process_owner) +``` + +We also use Fabric to automate some of the more complex things we need to do. + +This task `curl`'s the site's home page to make sure we haven't completely borked +things. We use it in lots of other tasks as a sanity check. + +```python +def check(): + '''Check that the home page of the site returns an HTTP 200.''' + require('site_url') + + print('Checking site status...') + + if not '200 OK' in local('curl --silent -I "%s"' % env.site_url, capture=True): + _sad() + else: + _happy() +``` + +The `_happy` and `_sad` functions just print out some simple messages to get our +attention: + +```python +from fabric.colors import red, green + +def _happy(): + print(green('\nLooks good from here!\n')) + +def _sad(): + print(red(r''' + ___ ___ + / /\ /__/\ + / /::\ \ \:\ + / /:/\:\ \__\:\ + / /:/ \:\ ___ / /::\ + /__/:/ \__\:\ /__/\ /:/\:\ + \ \:\ / /:/ \ \:\/:/__\/ + \ \:\ /:/ \ \::/ + \ \:\/:/ \ \:\ + \ \::/ \ \:\ + \__\/ \__\/ + ___ ___ ___ ___ + /__/\ / /\ / /\ / /\ ___ + \ \:\ / /::\ / /:/_ / /:/_ /__/\ + \ \:\ / /:/\:\ / /:/ /\ / /:/ /\ \ \:\ + _____\__\:\ / /:/ \:\ / /:/ /:/_ / /:/ /::\ \ \:\ + /__/::::::::\ /__/:/ \__\:\ /__/:/ /:/ /\ /__/:/ /:/\:\ \ \:\ + \ \:\~~\~~\/ \ \:\ / /:/ \ \:\/:/ /:/ \ \:\/:/~/:/ \ \:\ + \ \:\ ~~~ \ \:\ /:/ \ \::/ /:/ \ \::/ /:/ \__\/ + \ \:\ \ \:\/:/ \ \:\/:/ \__\/ /:/ __ + \ \:\ \ \::/ \ \::/ /__/:/ /__/\ + \__\/ \__\/ \__\/ \__\/ \__\/ + + + Something seems to have gone wrong! + You should probably take a look at that. + ''')) +``` + +This one is for when `python manage.py reset APP` is broken because you've changed +some `db_column` names and Django chokes because of some constraits and you just want +to **reset the fucking app**. + +It's the "NUKE IT FROM ORBIT!!" option. + +```python +def KILL_IT_WITH_FIRE(app): + require('site_path') + require('venv_path') + + with cd(env.site_path): + # Generate and download the reset SQL. + sudo(_python('manage.py sqlreset %s > reset.orig.sql' % app)) + get('reset.orig.sql') + + with open('reset.sql', 'w') as f: + with open('reset.orig.sql') as orig: + # Step through the first chunk of the file (the "drop" part). + line = orig.readline() + while not line.startswith('CREATE'): + if 'CONSTRAINT' in line: + # Don't write out CONSTRAINT lines. + # They're a problem when you change db_colum names. + pass + elif 'DROP TABLE' in line: + # Cascade drops. + # Hence with "with fire" part of this task's name. + line = line[:-2] + ' CASCADE;\n' + f.write(line) + else: + # Write other lines through untoched. + f.write(line) + line = orig.readline() + + # Write out the rest of the file untouched. + f.write(line) + f.write(orig.read()) + + # Upload the processed SQL file. + put('reset.sql', os.path.join(env.site_path, 'reset.ready.sql'), use_sudo=True) + + with cd(env.site_path): + # Use the SQL to reset the app, and fake a migration. + run(_python('manage.py dbshell < reset.ready.sql')) + sudo(_python('manage.py migrate --fake --delete-ghost-migrations ' + app)) +``` + +This task uses Mercurial's local tags to add a `production` or `staging` tag in your local +repository, so you can easy see where the production/staging servers are at +compared to your local repo. + +```python +def retag(): + '''Check which revision the site is at and update the local tag. + + Useful if someone else has deployed (which makes your production/staging local + tag incorrect. + ''' + require('site_path', provided_by=['prod', 'stag']) + require('env_name', provided_by=['prod', 'stag']) + + with cd(env.site_path): + current = run('hg id --rev . --quiet').strip(' \n+') + + local('hg tag --local --force %s --rev %s' % (env.env_name, current)) +``` + +This task tails the Gunicorn logs on the server so you can quickly find out what's +happening when things blow up. + +```python +def tailgun(follow=''): + """Tail the Gunicorn log file.""" + require('site_path') + + with cd(env.site_path): + if follow: + run('tail -f .gunicorn.log') + else: + run('tail .gunicorn.log') +``` + +We've got a lot of other tasks but they're pretty specific to our setup. + +Wrangling Databases with South +------------------------------ + +If you're not using [South][], you need to start. Now. + +No, really, I'll wait. Take 30 minutes, try the [tutorial][Southtut], wrap your head +around it and come back. It's far more important than this blog post. + +[South]: http://south.aeracode.org/ +[Southtut]: http://south.aeracode.org/docs/tutorial/index.html + +### Useful Fabric Tasks + +South is awesome but its commands are very long-winded. Here's the set of fabric +tasks I use to save quite a bit of typing: + +```python +def migrate(args=''): + '''Run any needed migrations.''' + require('site_path') + require('venv_path') + + with cd(env.site_path): + sudo(_python('manage.py migrate ' + args)) + +def migrate_fake(args=''): + '''Run any needed migrations with --fake.''' + require('site_path') + require('venv_path') + + with cd(env.site_path): + sudo(_python('manage.py migrate --fake ' + args)) + +def migrate_reset(args=''): + '''Run any needed migrations with --fake. No, seriously.''' + require('site_path') + require('venv_path') + + with cd(env.site_path): + sudo(_python('manage.py migrate --fake --delete-ghost-migrations ' + args)) +``` + +Remember that running a migration without specifying an app will migrate everything, +so a simple `fab dev migrate` will do the trick. + +Watching for Changes +--------------------- + +When developing locally you'll want to make a change to your code and have the server +reload that code automatically. The Django development server does this, and we can +hack it into our Vagrant/Gunicorn setup too. + +First, add a `monitor.py` file at the root of your project (I believe I found this +code [here][monitor], but I may be wrong): + +```python +import os +import sys +import time +import signal +import threading +import atexit +import Queue + +_interval = 1.0 +_times = {} +_files = [] + +_running = False +_queue = Queue.Queue() +_lock = threading.Lock() + +def _restart(path): + _queue.put(True) + prefix = 'monitor (pid=%d):' % os.getpid() + print >> sys.stderr, '%s Change detected to \'%s\'.' % (prefix, path) + print >> sys.stderr, '%s Triggering process restart.' % prefix + os.kill(os.getpid(), signal.SIGINT) + +def _modified(path): + try: + # If path doesn't denote a file and were previously + # tracking it, then it has been removed or the file type + # has changed so force a restart. If not previously + # tracking the file then we can ignore it as probably + # pseudo reference such as when file extracted from a + # collection of modules contained in a zip file. + + if not os.path.isfile(path): + return path in _times + + # Check for when file last modified. + + mtime = os.stat(path).st_mtime + if path not in _times: + _times[path] = mtime + + # Force restart when modification time has changed, even + # if time now older, as that could indicate older file + # has been restored. + + if mtime != _times[path]: + return True + except: + # If any exception occured, likely that file has been + # been removed just before stat(), so force a restart. + + return True + + return False + +def _monitor(): + while 1: + # Check modification times on all files in sys.modules. + + for module in sys.modules.values(): + if not hasattr(module, '__file__'): + continue + path = getattr(module, '__file__') + if not path: + continue + if os.path.splitext(path)[1] in ['.pyc', '.pyo', '.pyd']: + path = path[:-1] + if _modified(path): + return _restart(path) + + # Check modification times on files which have + # specifically been registered for monitoring. + + for path in _files: + if _modified(path): + return _restart(path) + + # Go to sleep for specified interval. + + try: + return _queue.get(timeout=_interval) + except: + pass + +_thread = threading.Thread(target=_monitor) +_thread.setDaemon(True) + +def _exiting(): + try: + _queue.put(True) + except: + pass + _thread.join() + +atexit.register(_exiting) + +def track(path): + if not path in _files: + _files.append(path) + +def start(interval=1.0): + global _interval + if interval < _interval: + _interval = interval + + global _running + _lock.acquire() + if not _running: + prefix = 'monitor (pid=%d):' % os.getpid() + print >> sys.stderr, '%s Starting change monitor.' % prefix + _running = True + _thread.start() + _lock.release() +``` + +Next add a `post_fork` hook to your Gunicorn config file that uses the monitor to +watch for changes: + +```python +def post_fork(server, worker): + import monitor + import local_settings + if local_settings.DEBUG: + server.log.info("Starting change monitor.") + monitor.start(interval=1.0) +``` + +Now the Gunicorn server will automatically restart whenever code is changed. Use +whatever method for determining debug status that you like. We use +`local_settings.py` files which all have `DEBUG` variables, so that works for us. + +It will *not* restart when you add new code (e.g. when you install a new app), so +you'll need to handle that manually with `fab dev restart`, but that's not too bad! + +[monitor]: http://code.google.com/p/modwsgi/wiki/ReloadingSourceCode + +### Using the Werkzeug Debugger with Gunicorn + +The final piece of the puzzle is being able to use the fantastic [Werkzeug +Debugger][debug] while running on the development VM with Gunicorn. + +To do this, create a `debug_wsgi.py` file at the root of your project: + +```python +import os +import sys +import site + +parent = os.path.dirname +site_dir = parent(os.path.abspath(__file__)) +project_dir = parent(parent(os.path.abspath(__file__))) + +sys.path.insert(0, project_dir) +sys.path.insert(0, site_dir) + +site.addsitedir('VIRTUALENV_SITE_PACKAGES') + +from django.core.management import setup_environ +import settings +setup_environ(settings) + +import django.core.handlers.wsgi +application = django.core.handlers.wsgi.WSGIHandler() + +from werkzeug.debug import DebuggedApplication +application = DebuggedApplication(application, evalex=True) + +def null_technical_500_response(request, exc_type, exc_value, tb): + raise exc_type, exc_value, tb +from django.views import debug +debug.technical_500_response = null_technical_500_response +``` + +Have Gunicorn use this file to run your development server with `gunicorn +debug_wsgi:application`. + +Make sure to replace `'VIRTUALENV_SITE_PACKAGES'` with the _full_ path to your +virtualenv's `site_packages` directory. You might want to make this a setting in +a machine-specific settings file. + +[debug]: http://werkzeug.pocoo.org/docs/debug/ + +### Pulling Uploads + +Once you give a client access to a site they'll probably be uploading images (through +Django's built-in file uploading features or with [django-filebrowser][]). + +When you're making changes locally it's often useful to have these uploaded files on +your local VM, otherwise you end up with a bunch of broken images. + +Here's a simple Fabric task that will pull down all the uploads from the server: + +```python +def pull_uploads(): + '''Copy the uploads from the site to your local machine.''' + require('uploads_path') + + sudo('chmod -R a+r "%s"' % env.uploads_path) + + rsync_command = r"""rsync -av -e 'ssh -p %s' %s@%s:%s %s""" % ( + env.port, + env.user, env.host, + env.uploads_path.rstrip('/') + '/', + 'media/uploads' + ) + print local(rsync_command, capture=False) +``` + +You might be wondering about the line that strips `/` characters and then adds them +back in. `rsync` does different things depending on whether you end a path with +a `/`, so this is actually pretty important. + +In your host task you'll need to set the `uploads_path` variable to something like +this: + +```python +import os +env.site_path = os.path.join('var', 'www', 'myproject') +env.uploads_path = os.path.join(env.site_path, 'media', 'uploads') +``` + +Now you can run `fab production pull_uploads` to pull down all the files people have +uploaded to the production server. + +[django-filebrowser]: http://code.google.com/p/django-filebrowser/ + +### Preventing Accidents + +Deploying to test and staging servers should be quick and easy. Deploying to +production servers should be harder to prevent people from accidentally doing it. + +I've created a little function that I call before deploying to production servers. +It forces me to type in random words from the system word list before proceeding to +make sure I *really* know what I'm doing: + +```python +import os, random + +from fabric.api import * +from fabric.operations import prompt +from fabric.utils import abort + +WORDLIST_PATHS = [os.path.join('/', 'usr', 'share', 'dict', 'words')] +DEFAULT_MESSAGE = "Are you sure you want to do this?" +WORD_PROMPT = ' [%d/%d] Type "%s" to continue (^C quits): ' + +def prevent_horrible_accidents(msg=DEFAULT_MESSAGE, horror_rating=1): + """Prompt the user to enter random words to prevent doing something stupid.""" + + valid_wordlist_paths = [wp for wp in WORDLIST_PATHS if os.path.exists(wp)] + + if not valid_wordlist_paths: + abort('No wordlists found!') + + with open(valid_wordlist_paths[0]) as wordlist_file: + words = wordlist_file.readlines() + + print msg + + for i in range(int(horror_rating)): + word = words[random.randint(0, len(words))].strip() + p_msg = WORD_PROMPT % (i+1, horror_rating, word) + answer = prompt(p_msg, validate=r'^%s$' % word) +``` + +You may need to adjust `WORDLIST_PATHS` if you're not on OS X. + +Working with Third-Party Apps +----------------------------- + +One of the best parts about working with Django is that many problems have already +been solved and the solutions have been released as open-source applications. + +We use quite a few open-source apps, and there are a couple of tricks I've learned to +make working with them easier. + +### Installing Apps from Repositories + +If I'm going to use an open-source Django app in a project I'll almost always install +it as an editable repository on the VM with `pip install -e`. + +Others may disagree with me on this, but I think it's the best way to work. + +Often I'll find a bug that I think may be in one of the third-party apps I'm using. +Installing the apps as repositories makes it easy to read their source and figure out +if the bug is really in the app. + +If the bug *is* in the third-party app having the app installed as a repository makes +it simple to fix the bug, fork the project on BitBucket or GitHub, send a pull +request, and get back to work. + +### Mirroring Repositories + +One problem we've run into at Dumbwaiter is that the repos for third-party apps we +use are scattered across GitHub, BitBucket, Google Code, and other servers. If any +one of these services goes down we're stuck waiting for it to come back up. + +A while ago I took half a day and consolidated all of these repos onto one of the +servers that we control. The basic process went like this: + +* Use [hg-git][] and [hgsubversion][] to convert the git and SVN repos to Mercurial + repos. +* Set up a master `mirror` Mercurial repo with all the app repos as subrepos. +* Push the master repo and all the subrepos up to one of our Linodes. + +Now we can use `-e ssh://hg@OUR_LINODE/mirror/APP@REV_THAT_WORKS#egg=APP` in our +`requirements.txt` files to install apps from our mirror. When we want to update our +dependencies we can simply pull from the upstream repos and commit in the mirror +repo. + +If our mirror goes down it's not a big deal, because we have far bigger problems to +worry about than new projects. + +I wrote a few scripts to automate updating apps and such, but they're extremely hacky +so I don't want to post them here. Take half a day and write your own set — it's +definitely worth it to have your own mirror of your specific dependencies. + +[hg-git]: http://hg-git.github.com/ +[hgsubversion]: https://bitbucket.org/durin42/hgsubversion/wiki/Home + +### Using BCVI to Edit Files + +I said that when I find a bug that I think is in a third-party app I'll poke around +with the app and try to figure it out. But since all the apps are installed in +a virtualenv on the Vagrant VM it might seem like it's a pain in the ass to edit +those files! + +Luckily [BCVI][] exists. It's a utility that opens a "back channel" to your local +machine when you SSH and lets you run `vi FILE` to open that file in +Vim/MacVim/GVim/etc on your *local* machine. When you save the file it uploads it +back to the server automatically for you. + +It can be a bit tricky to set up, but it's worth it. Trust me. + +[BCVI]: http://sshmenu.sourceforge.net/articles/bcvi/ + +Improving the Admin Interface +----------------------------- + +I'm going to be honest: Django's admin interface is the main reason I'm still using +it. Other frameworks like [Flask][] are great, but Django's admin saves me +*ridiculous* amounts of time when I'm making simple CRUD sites for clients. + +That said, the Django admin isn't the prettiest thing around, but we can give it +a facelift. + +[Flask]: http://flask.pocoo.org/ + +### Enter Grappelli + +[Grappelli][] is a Django app that reskins the admin interface beautifully. It also +adds some functionality like drag-and-drop reordering of inlines, and allows you to +customize the dashboard to your liking. *Every* Django site I work on uses Grappelli +-- it's just that good. + +The downside of Grappelli is that it changes quite a lot and breaks backwards +compatibility at the drop of a hat. + +If you're going to use Grappelli you *must* freeze your requirements.txt files and +work with a single version at a time. Trying to always work from the trunk will make +you drink. + +[Grappelli]: http://django-grappelli.readthedocs.org/ + +### An Ugly Hack to Show Usable Foreign Key Fields + +A limitation of both Grappelli and the stock Django admin is that it seems like you +can't easily show fields from related models in the admin list view. + +For example, if you're new to Django you might expect this to work: + +```python +class BlogEntryAdmin(admin.ModelAdmin): + list_display = ('title', 'author__name') +``` + +Unfortunately Django chokes on the `author__name` lookup. You can *display* the name +without too much fuss: + +```python +class BlogEntryAdmin(admin.ModelAdmin): + list_display = ('title', 'author_name') + + def author_name(self, obj): + return obj.name +``` + +That will display the name just fine. However, it won't be a fully-fledged column in +the Django admin because you can't sort on it. + +It may seem like this is the end — if it could be a fully-functional field, why +wouldn't Django just let you use `author__name`? Luckily we can add one more line to +fix the problem: + +```python +class BlogEntryAdmin(admin.ModelAdmin): + list_display = ('title', 'author_name') + + def author_name(self, obj): + return obj.name + author_name.admin_order_field = 'author__name' +``` + +Now the author name has all the functionality of a real `list_display` entry. + +Using Django-Annoying +--------------------- + +If you haven't heard of [django-annoying][] you should definitely check it out. It's +got a bunch of miscellaneous functions that fix some common, annoying parts of +Django. + +My two personal favorites from the package are a pair of decorators that help make +your views much, much cleaner. + +[django-annoying]: https://bitbucket.org/offline/django-annoying/wiki/Home + +### The render\_to Decorator + +The decorator is called `render_to` and it eliminates the ugly `render_to_response` +calls that Django normally forces you to use in every single view. + +Normally you'd use something like this: + +```python +def videos(request): + videos = Video.objects.all() + return render_to_response('video_list.html', { 'videos': videos }, + context_instance=RequestContext(request)) +``` + +With `render_to` your view gets much cleaner: + +```python +@render_to('video_list.html') +def videos(request): + videos = Video.objects.all() + return { 'videos': videos } +``` + +Less typing `context_instance=...` over and over, and less syntax to remember. + +Yes, I know about Django 1.3's `render` shortcut. You have to type `request` every +single time with `render`, so the `render_to` decorator still wins. + +### The ajax\_request Decorator + +The `ajax_request` decorator is like `render_to` for AJAX requests. You simply +return a Python dictionary from your view and the decorator handles the JSON encoding +and such: + +```python +@ajax_request +def ajax_get_entries(request): + blog_entries = BlogEntry.objects.all() + return { 'entries': [(entry.title, entry.get_absolute_url()) + for entry in entries]} +``` + +Templating Tricks +----------------- + +I'm not a frontend developer, but I've done my share of HTML hacking at Dumbwaiter. +Here are a few of the tricks I've learned. + +### Null Checks and Fallbacks + +A common pattern I see in Django templates looks like this: + +```django +{% templatetag openblock %} if business.title {% templatetag closeblock %} + {% templatetag openvariable %} business.title {% templatetag closevariable %} +{% templatetag openblock %} else {% templatetag closeblock %} + {% templatetag openvariable %} business.short_title {% templatetag closevariable %} +{% templatetag openblock %} endif {% templatetag closeblock %} +``` + +Here's a simpler way to do that: + +```django +{% templatetag openblock %} firstof business.title business.short_title {% templatetag closeblock %} +``` + +`firstof` will return the first non-Falsy item in its arguments. + +### Manipulating Query Strings + +Query strings are normally not a big deal, but every once in a while you'll have +a model listing page where you need to filter by category, and number of spaces, and +tags, etc all at once. + +If you're trying to manage GET queries manually it can get pretty hairy very fast. + +[This Django snippet][qstring] makes working with query strings in templates +a breeze. + +[qstring]: http://djangosnippets.org/snippets/2237/ + +### Satisfying Your Designer with Typogrify + +If you haven't heard of [Typogrify][] you should take a look at it. It makes it easy +to add all the typographic goodness your designers are looking for. + +[Typogrify]: http://code.google.com/p/typogrify/ + +The Flat Page Trainwreck +------------------------ + +Creating a site for a client is very different than creating a site for yourself. +For pretty much every client we've dealt with we've heard: "can't we just create +a new page at /drink-special/ for this special deal we're running?" + +Having clients go through you to make new pages is simply too much overhead. We +needed a way to let clients create new pages (like `/drink-special/`) on the fly, +without our intervention. + +Django has a "flatpages" app that solves this problem. Kind of. + +When using flat pages clients need to do two things that are often too much for +non-technical people: + +* Manage URLs manually. +* Write all content as raw HTML in a single text field. + +We've tried a lot of Django CMS apps at Dumbwaiter, and none of them made us happy. +They all seemed to have some or all of the following problems: + +* They take over your site and make you write a "Django-WhateverCMS site" instead of + a "Django site". +* They're extremely feature-rich and complicated with features like + internationalization, redirects, versions, and many others. This is great if you + need the flexibility, but bad if your clients just need to create a couple of + pages. +* They break `APPEND_TRAILING_SLASH` and make you clutter your `urls.py` files with + a bunch of extra code ot handle this. + +I finally got fed up and wrote my own Django CMS app: [Stoat][]. Stoat is designed +to be sleek, with only the features that our clients need. + +It's not officially version 1.0 yet, but we're using it for a few clients and it's +working well. Check it out if you're looking for a more lightweight CMS app. + +[Stoat]: http://stoat.rtfd.org/ + +Editing with Vim +---------------- + +I [use Vim][vimpost] to edit everything. Naturally I've found a bunch of plugins, +mappings and other tricks that make it even better when working on Django projects. + +[vimpost]: /blog/2010/09/coming-home-to-vim/ + +### Vim for Django + +There are a lot of ways to make Vim work with Django. I won't go into all of them in +this post, but a good place to start is [this Django wiki page][vimdjango]. + +[vimdjango]: https://code.djangoproject.com/wiki/UsingVimWithDjango + +### Filetype Mappings + +Most files in a Django project have one of two extensions: `.py` and `.html`. +Unfortunately these extensions aren't unique to Django, so Vim doesn't automatically +set the correct `filetype` when you open one. + +I've added a few mappings to my `.vimrc` to make it quick and easy to set the correct +`filetype`: + +```vim +nnoremap _dt :set ft=htmldjango +nnoremap _pd :set ft=python.django +``` + +I also have a few autocommands that set the filetype for me when I'm editing a file +whose name "sounds like" a Django file: + +```vim +au BufNewFile,BufRead admin.py setlocal filetype=python.django +au BufNewFile,BufRead urls.py setlocal filetype=python.django +au BufNewFile,BufRead models.py setlocal filetype=python.django +au BufNewFile,BufRead views.py setlocal filetype=python.django +au BufNewFile,BufRead settings.py setlocal filetype=python.django +au BufNewFile,BufRead forms.py setlocal filetype=python.django +``` + +### Python Sanity Checking + +Lets be honest here: it takes a lot of work to turn Vim into an "IDE", and even then +it doesn't reach the level of something like Eclipse for Java. Anyone who claims it +has the same levels of integration and functionality is simply lying. + +With that said I'll make an opinionated statement that is going to piss some of you +off. + +**I am a programmer, not an IDE operator.** + +I know Python. + +I know Django. + +I don't need to hit Cmd+Space twice for every line of code I write. + +When someone asks me "how do you run your site" I do **not** answer: "click the green +triangle in Eclipse". + +However, I am human. I do stupid things like forgetting a colon or forgetting an +import. To help me with those problems I've turned to [Syntastic][] and Kevin +Watters' [Pyflakes fork][] for Vim. + +Syntastic is a Vim plugin that adds on-the-fly syntax-checking for many different +file formats. If you have Pyflakes installed it will automatically show you errors +in your code. + +Pyflakes doesn't have IDE-level integration with your code. It doesn't check that +whatever libraries you `import` actually exist. It simply checks that your files are +probably-valid Python, and tells you when they're not. + +This is enough for me. It catches the stupid mistakes I make. The less-stupid, +more-subtle mistakes slip by it, but to be fair many of them would have slipped by an +"IDE" as well. + +[Pyflakes fork]: https://github.com/kevinw/pyflakes +[Syntastic]: http://www.vim.org/scripts/script.php?script_id=2736 + +### Javascript Sanity Checking and Folding + +Syntastic also supports Javascript if you have Javascript Lint installed (`brew +install jsl` on OS X). It's not perfect but it *will* catch things like using +trailing commas in object literals. + +Some people like using CTags to get an overview of their code. I take a more +low-tech approach and am in love with code folding. When I fold my code +I automatically get an overview of everything in each file. + +By default Vim doesn't fold Javascript files, but you can add some basic, perfectly +serviceable folding with these two lines in your .vimrc: + +```vim +au FileType javascript setlocal foldmethod=marker +au FileType javascript setlocal foldmarker={,} + +``` + +### Django Autocommands + +I *rarely* work with raw HTML files any more. Whenever I open a file ending in +`.html` it's almost always a Django template (or a [Jinja][] template, which has +a very similar syntax). I've added an autocommand to automatically set the correct +filetype whenever I open a `.html` file: + +[Jinja]: http://jinja.pocoo.org/ + +```vim +au BufNewFile,BufRead *.html setlocal filetype=htmldjango +``` + +I also have some autocommands that tweak how a few specific files are handled: + +```vim +au BufNewFile,BufRead urls.py setlocal nowrap +au BufNewFile,BufRead settings.py normal! zR +au BufNewFile,BufRead dashboard.py normal! zR +``` + +This automatically unfolds `urls.py`, `dashboard.py` and `settings.py` (I prefer +seeing those unfolded) and unsets line wrapping for `urls.py` (lines in a `urls.py` +file can get long and are hard to read when wrapped). + +Conclusion +---------- + +I hope that this longer-than-expected blog entry has given you at least one or two +things to think about. + +I've learned a lot while working with Django for Dumbwaiter every day, but I'm sure +there's still a lot I've missed. If you see something I could be doing better please +let me know! + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2011/09/writing-vim-plugins.html --- a/content/blog/2011/09/writing-vim-plugins.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,599 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "Writing Vim Plugins" - snip: "It's pretty much black magic." - created: 2011-09-06 09:13:00 - %} - - {% block article %} - - -A while ago I wrote a [post][vimpost] about switching back to [Vim][]. Since then -I've written two plugins for Vim, one of which has been officially "released". - -A couple of people have asked me if I'd write a guide to creating Vim plugins. -I don't feel confident enough to write an official "guide", but I do have some advice -for Vim plugin authors that might be useful. - -[vimpost]: http://stevelosh.com/blog/2010/09/coming-home-to-vim/ -[Vim]: http://www.vim.org/ - -[TOC] - -Other People Who Know More Than I Do ------------------------------------- - -Writing two decently-sized Vim plugins has given me some experience, but there are -a lot of people that know far more than I do. There are two in particular that come -to mind. I'd love for them to write some guides (or even books) about modern-day Vim -scripting. - -### Tim Pope - -The first is [Tim Pope][]. He's written a ton of Vim plugins like [Pathogen][], -[Surround][], [Repeat][], [Speeddating][] and [Fugitive][]. Each of those is clear, -focused and polished. - -It would be awesome to read a guide on the ins and outs of Vim scripting by him. - -[Tim Pope]: http://tpo.pe/ -[Pathogen]: https://github.com/tpope/vim-pathogen -[Surround]: https://github.com/tpope/vim-surround -[Repeat]: https://github.com/tpope/vim-repeat -[Speeddating]: https://github.com/tpope/vim-speeddating -[Fugitive]: https://github.com/tpope/vim-fugitive - -### Scrooloose - -The other person that comes to mind is [Scrooloose][], author of [NERDTree][], -[NERDCommenter][] and [Syntastic][]. - -His plugins are large and full-featured but work incredibly well, considering how -tricky and painful Vimscript is to work with. I'd love to read a guide on writing -large-scale Vim plugins by him. - -[Scrooloose]: http://got-ravings.blogspot.com/ -[NERDTree]: https://github.com/scrooloose/nerdtree -[NERDCommenter]: https://github.com/scrooloose/nerdcommenter -[Syntastic]: https://github.com/scrooloose/syntastic - -Be Pathogen-Compatible ----------------------- - -It's 2011. When writing your plugin, *please* make its source compatible with -[Pathogen][]. It's very easy to do this -- just set up your project's files like -this: - - :::text - yourplugin/ - doc/ - yourplugin.txt - plugin/ - yourplugin.vim - ... - README - LICENSE - -This will let users use Pathogen (or [Vundle][]) to install and use your plugin. - -The days of "unzip and drag the files into the right directories" and the horror of -Vimballs are over. Pathogen and Vundle are the right way to manage plugins, so let -your users use them. - -[Vundle]: https://github.com/gmarik/vundle - -Please, For the Love of God, Use normal! ----------------------------------------- - -My first piece of actual scripting advice is something simple but important. If -you're writing a Vim plugin and need to perform some actions, you might be tempted to -use `normal`. Don't. Instead, you need to use `normal!`. - -`normal!` is like `normal`, but ignores mappings the user has set up. If you use -plain old `normal dd` and I've remapped `dd` to do something else, the call will use -my mapping and probably not do what your plugin expects. Using `normal!` ensures -that the call will do what you expect no matter what the user has mapped. - -This is a single instance of a more general theme. Vim is very customizable and users -will do lots of crazy things in their `.vimrc` files. If a key can be mapped or -a setting changed, you *have* to assume that some user of your plugin will have -mapped or changed it. - -Mapping Keys the Right Way --------------------------- - -Most plugins add key mappings to make them easier to use. Unfortunately this can be -tricky to get right. You can never tell what keys your users have already mapped -themselves, and shadowing someone's favorite key mapping will break their muscle -memory and annoy them to no end. - -### When to Map Keys - -The first question to ask is whether your plugin needs to map keys itself at all. - -My [Gundo][] plugin has only one feature that needs to be mapped to a key in order to -make it useful: the "toggle Gundo" action. - -Gundo doesn't map this key itself, because no matter what "default" mapping I pick -someone will have already mapped it. Instead I added a section right in the README -file that shows how a user can map the key themselves: - - :::vim - nnoremap :GundoToggle - -By making users add this line to their `.vimrc` themselves it shows them which key is -used to toggle Gundo (which they would have to know anyway) and also makes it obvious -how to change it to suit their taste. - -[Gundo]: http://sjl.bitbucket.org/gundo.vim/ - -### imap and nmap are Pure Evil - -Sometimes forcing the user to map their own keys won't work. Perhaps your plugin has -many mappings that would be tedious for a user to set up manually (like my -[Threesome][] plugin), or its mappings are mnemonic and wouldn't really make sense if -mapped to other keys. - -I'll talk more about how to deal with this in a moment, but the most important thing -to remember when mapping your own keys is that you must always, *always*, -***always*** use the `noremap` forms of the various `map` commands. - -If you map a key with `nmap` and the user has remapped a key that your mapping uses, -your mapped key will almost certainly not do what you want. Using `nnoremap` will -ignore user mappings and do what you expect. - -This is the same principle as `normal` and `normal!`: *never* trust your users' -configurations. - -[Threesome]: http://sjl.bitbucket.org/threesome.vim - -### Let Me Configure Mappings - -If you feel that your plugin must map some keys, please make those mappings -configurable in some way. - -There are a number of ways to do this. The easiest way is to provide a configuration -option that disables all mappings. The user can them remap the keys as they see fit. -For example: - - :::vim - if !exists('g:yourplugin_map_keys') - let g:yourplugin_map_keys = 1 - endif - - if g:yourplugin_map_keys - nnoremap d :call YourPluginDelete() - endif - -Normal users will get the mappings automatically set up for them, and power users can -remap the keys to whatever they wish to avoid shadowing their own mappings. - -If your plugin's mappings all start with a common prefix (like `` or -``) you have another option: allow users to configure this prefix. This -is the approach I've used in [Threesome][]. It works like this: - - :::vim - if !exists('g:yourplugin_map_prefix') - let g:yourplugin_map_prefix = '' - endif - - execute "nnoremap" g:yourplugin_map_prefix."d" ":call YourPluginDelete()" - -The `execute` command lets you build the mapping string dynamically so your users can -change the mapping prefix. - -There is a third option for solving this problem: the `hasmapto()` Vim function. -Some plugins will use this to map a command to a key *unless* the user has already -mapped that command to something else. I don't personally like this option because -it feels less clear to me, but I know other people feel differently so I wanted to -mention it. - -Localize Mappings and Settings ------------------------------- - -The next step in being a good Vim plugin author is to try to minimize the effects of -your key mappings and setting changes. Some plugins will need to have global -effects but others will not. - -For example: if you're writing a plugin for working with Python files it should only -take effect for Python buffers, not all buffers. - -### Localizing Mappings - -Key binding are easy to localize to single buffers. All of the `noremap` commands -can take an extra `` argument that will localize the mapping to the current -buffer. - - :::vim - " Remaps z globally - nnoremap z :YourPluginFoo - - " Remaps z only in the current buffer - nnoremap z :YourPluginFoo - -However, the problem is that you need to run this command in every buffer you want -the mapping active. To do this your plugin can use an `autocommand`. Here's a full -example, using this concept plus the previously mentioned configuration options: - - if !exists('g:yourplugin_map_keys') - let g:yourplugin_map_keys = 1 - endif - - if !exists('g:yourplugin_map_prefix') - let g:yourplugin_map_prefix = '' - endif - - if g:yourplugin_map_keys - execute "autocommand FileType python" "nnoremap " g:yourplugin_map_prefix."d" ":call YourPluginDelete()" - endif - -Now your plugin will define a key mapping only for Python buffers, and your users can -disable or customize this mapping as they see fit. - -This mapping command is quite ugly. Unfortunately that's the price of using -Vimscript and trying to make a plugin that will work for many users. Later I'll talk -about one possible solution to this ugliness. - -### Localizing Settings - -Just as you should make mappings local to buffers when appropriate, you should do the -same with settings like `foldmethod`, `foldmarker` and `shiftwidth`. Not all -settings can be set locally in a buffer. You can read `:help ` to see -if it's possible. - -You can use `setlocal` instead of `set` to localize settings to individual buffers. -Like with mappings you'll need to use an autocommand to run the `setlocal` command -every time the users opens a new buffer. - -Autoload is Your Friend ------------------------ - -If your plugin is something that users will be using all the time you can skip this -section. - -If you're writing something that will only be used in specific cases, you can help -your users by using Vim's `autoload` functionality to delay loading its code until -the user actually tries to use it. - -The way `autoload` works is fairly simple. Normally you would bind a key to call one of your -plugin's functions with something like this: - - :::vim - nnoremap z :call YourPluginFunction() - -You can use autoloading by prepending `yourplugin#` to the name of the function: - - :::vim - nnoremap z :call yourplugin#YourPluginFunction() - -When this mapping is run, Vim will do the following: - -1. Check to see if `YourPluginFunction` is already defined. If so, call it. -2. Otherwise, look in `~/.vim/autoload/` for a file named `yourplugin.vim`. -3. If it exists, parse and load the file (which presumably defines - `YourPluginFunction` somewhere inside of it). -4. Call the function. - -This means that instead of putting all of your plugin's code in -`plugin/yourplugin.vim` you can put just the key mapping code there and pull the rest -out into `autoload/yourplugin.vim`. - -If your plugin has a decent amount of code this can reduce the startup time of Vim by -a significant amount. - -Check out the full documentation of `autoload` by running `:help autoload` to learn -much more. - -Backwards Compatibility is a Big Deal -------------------------------------- - -Once you've written your Vim plugin and released it into the wild, you have to -maintain it. Users will find bugs and ask for new features. - -Part of being a responsible developer of any kind, including a Vim plugin author, is -maintaining backwards compatibility, *especially* for tools that users will use every -day and burn into their muscle memory. Users rely on tools to work, and tools that -break backwards compatibility will quickly lose users' trust. - -Maintaining backwards compatibility will cause your plugin's code to get crufty in -spots, but it's the price of maintaining your users' happiness. - -### What Matters for Backards Compatibility? - -For a Vim plugin the most important part of staying backwards compatible is ensuring -that key mappings, customized or not, continue to do what users expect. - -If your plugin maps key `X` to do `Y`, then pressing `X` should *always* do `Y`, even -if you change how `Y` is called by renaming `Y` to `Z`. This may mean changing `Y` -into a wrapper function which simply calls `Z`. - -There are many other aspects of backwards compatibility that you will have to -consider, depending on the purpose of your plugin. The rule of thumb you should -follow is: if a user uses this plugin on a daily basis and has its usage burned into -their muscle memory, updating the plugin should not make them relearn anything. - -### Use Semantic Versioning So I Can Stay Sane - -A fast, simple, easy way to document your plugin's state is to use [semantic -versioning][]. - -Semantic versioning is simply the idea that instead of picking arbitrary version -numbers for releases of your project, you use version numbers that describe the -backwards-compatible state in a meaningful way. - -In a nutshell, these rules describe how you should select version numbers for new -releases: - -* Version numbers have three components: `major.minor.bugfix`. For example: `1.2.4` - or `2.13.0`. -* Versions with a major version of 0 (e.g. `0.2.3`) make no guarantees about - backwards compatibility. You are free to break anything you want. It's only after - you release `1.0.0` that you begin making promises. -* If a release introduces backwards-incompatible changes, increment the major version - number. -* If a release is backwards-compatible, but adds *new* features, increment the minor - version number. -* If a release simply fixes bugs, refactors code, or improves performance, increment - the bugfix version number. - -This simple scheme makes it easy for users to tell (in a broad sense) what has -changed when they update your project. - -If only the bugfix number has changed they can update without fear and continue on -without worrying about changes unless they're curious. - -If the minor version number has changed they might want to look at the changelog to -see what new features they may want to take advantage of, but if they're busy they -can simply update and move on. - -If the major version number has changed it's a major red flag, and they'll want to -read the changelog carefully to see what is different. - -Some people don't like semantic versioning for the following reason: - -> If I have to increment the major version number every time I make -> backwards-incompatible changes, I'll quickly be at ugly versions like 24.1.2! - -To this I say: "Yes, but if that happens you're doing things wrong in the first -place." - -Keep your project in "beta" (i.e. version `0.*.*`) for as long as you need to -experiment freely. *Take your time* and make sure you've gotten things (mostly) -right. Once you release `1.0.0` it's time to start being responsible and caring -about backwards compatibility. - -Breaking functionality all the time harms your users by reducing their productivity -and frustrating them. Yes, it means adding some cruft to your code over time, but -it's the price of not being evil. - -[semantic versioning]: http://semver.org/ - -Document Everything -------------------- - -A critical part of releasing a Vim plugin to the world is writing documentation for -it. Vim has fantastic documentation itself, so your plugins should follow in its -footsteps and provide thorough docs. - -### Pick Some Requirements and Stick to Them - -The most important part of your documentation is telling users what they need to have -in order to use your plugin. Vim runs on nearly every system imaginable and can be -compiled in many different ways, so being specific about your plugin's requirements -will save users a lot of trial and error. - -* Does your plugin only work with Vim version X.Y or later? -* Does it require Python/Ruby/etc support compiled in? Which version? -* Does it not work on Windows? -* Does it rely on an external tool? - -If the answer to any of those questions is "yes", you *must* mention it in the -documentation. - -### Write a README - -The first step to documenting your plugin is to write a README file for the -repository. You can also use the text of this file as the description if you upload -your plugin to the [vim website][], or the content of your plugin's website if you -create one for it. - -Some examples of things to include in your README are: - -* An overview of what the plugin does. -* Screenshots, if possible. -* Requirements. -* Installation instructions. -* Common configuration options that many users will want to know. -* Links to: - * A canonical web address to find the plugin. - * The bug tracker for the plugin. - * The source code or repository of the plugin. - -[vim website]: http://www.vim.org/ - -### Create a Simple Website - -This isn't strictly necessary, but having a simple website for your plugin is an -extra touch that makes it seem more polished. - -It also gives you a canonical URL that people can visit to get the latest information -about your plugin. - -I've made simple sites for both of my plugins: [Gundo][] and [Threesome][]. Feel -free to use them as an example or even take their code and use it for your own plugin -sites if you like. - -### Write a Vim Help Document - -The bulk of your plugin's documentation should be in the form of a Vim help document. -Users are used to using Vim's `:help` and they'll expect to be able to use it to -learn about your plugin. - -Creating a help document is as easy as creating a `doc/yourplugin.txt` file in your -project. It will be indexed automatically by `pathogen#helptags()` so your users -will have the docs at their fingertips. - -Two easy ways to learn the syntax of help files are by reading `:help help-writing` -and using an existing plugin's help file as an example. - -Take your time and craft a beautiful help file you can be proud of. Don't be afraid -to add a bit of personality to your docs to break the dryness. The [syntastic help -file][syntastic help] is a great example (especially the `About` section). - -Things to include in your documentation: - -* A brief overview of the plugin. -* A more in-depth description of how the plugin is used. -* Every single key mapping the plugin creates. -* Ways to extend the plugin, if applicable. -* All configuration variables (including their default values!). -* The plugin's changelog. -* The plugin's license. -* Links to the plugin's repository and bug tracker. - -In a nutshell: your help file should contain *anything* a user would ever need to know -about your plugin. - -[syntastic help]: https://github.com/scrooloose/syntastic/blob/master/doc/syntastic.txt - -### Keep a Changelog - -The last part of documenting your project is keeping a changelog. You can skip this -while your project is still in "beta" (i.e. less than version `1.0.0`) but once you -officially release a real version you need to keep your users informed about what has -changed between releases. - -I like to include this log in the README, the plugin's website, and the -documentation to make it as easy as possible for users to see what's changed. - -Try to keep the language of the changelog at a high enough level for your users to -understand without knowing anything about the implementation of your plugin. Things -like "added feature X" and "fixed bug Y" are great, while things like "refactored the -inner workings of utility function Z" are best left in commit messages. - -Making Vimscript Palatable --------------------------- - -The worst part about writing Vim plugins is, without a doubt, dealing with Vimscript. -It's an esoteric language that's grown organically over the years seemingly without -any strong design direction. - -Features are added to Vim, then Vimscript features are added to control those -features, then hacky workarounds are added for flexibility. - -The syntax is terse, ugly and inconsistent. Is `" foo` a comment? Sometimes. - -Much of the time you'll spend writing your first plugin will be learning how to do -things in Vimscript. The help documentation on all of its features is thorough, but -it can be hard to find what you're looking for if you don't know the exact name. -Looking through other plugins is often very helpful in pointing you toward what you -need. - -There are a couple of ways to ease the pain of Vimscript, and I'll briefly talk about -two of them here. - -### Wrap. Everything. - -The first piece of advice I have is this: if you want to make your plugins readable -and maintainable then you need to wrap up functionality even more than you would in -other languages. - -For example, my [Gundo][] plugin has a few utility functions that look like this: - - :::vim - function! s:GundoGoToWindowForBufferName(name)"{{{ - if bufwinnr(bufnr(a:name)) != -1 - exe bufwinnr(bufnr(a:name)) . "wincmd w" - return 1 - else - return 0 - endif - endfunction"}}} - -This function will go to the window for the given buffer name and gracefully handle -the case where the buffer/window does not exist. It's verbose but much more readable -than the alternative of using that `if` statement in every place I need to switch -windows. - -As you write your plugin you'll "grow" a number of these utility functions. Any time -you duplicate code you should think about creating one, but you should also do so any -time you write a particularly hairy line of Vimscript. Pulling complex lines out -into named functions will save you a lot of reviewing and rethinking down the line. - -### Scripting Vim with Other Languages - -Another option for making Vimscript less painful is to simply not use it much at all. -Vim includes support for creating plugins in a number of other languages like Python -and Ruby. Many plugin authors choose to move nearly all of their code into another -language, using a small Vimscript "wrapper" to expose it to the user. - -I decided to try this approach with [Threesome][] after seeing it used in the -[vim-orgmode][] plugin to great effect. Overall I consider it to be a good idea, -with a few caveats. - -First, using another language will requires your plugin's users to use a version of -Vim compiled with support for that version. In this day and age it's usually not -a problem, but if you want your plugin to run everywhere then it's not an option. - -Using another language adds overhead. You need to not only learn Vimscript but also -the interface between Vim and the language. For small plugins this can add more -complexity to the project than it saves, but for larger plugins it can pay for -itself. It's up to you to decide whether it's worth it. - -Finally, using another language does not entirely insulate you from the -eccentricities of Vimscript. You still need to learn how to do most things in -Vimscript -- using another language simply lets you wrap most of this up more neatly -than you otherwise could. - -[vim-orgmode]: https://github.com/jceb/vim-orgmode - -### Unit Testing Will Make You Drink - -Unit testing (and other types of testing) is becoming more and more popular today. -In particular the Python and Ruby communities seem to be getting more and more -excited about it as time goes on. - -Unfortunately, unit testing Vim plugins lies somewhere between "painful" and -"[garden-weasel][]ing your face" on the difficulty scale. - -I tried adding some unit tests to [Gundo][], but even after looking at a number of -frameworks I was spending hours simply trying to get my tests to function. - -I didn't even bother trying to add tests to [Threesome][] because for every hour -I would have spent fighting Vim to create tests I could have cleaned up the code and -fixed bugs instead. - -I'll gladly change my opinion on the subject if someone writes a unit testing -framework for Vim that's as easy to use as [Cram][]. In fact, I'll even buy the -author a $100 bottle of scotch (or whatever they prefer). - -Until that happens I personally don't think it's worth your time to unit test Vim -plugins. Spend your extra hours reading documentation, testing things manually with -a variety of settings, and thinking hard about your code instead. - -[garden-weasel]: http://www.amazon.com/Garden-Weasel-90206/dp/B002ECYRH4 -[Cram]: https://bitheap.org/cram/ - -TL;DR ------ - -Writing Vim plugins is tricky. Vimscript is a rabbit hole of sadness and despair, -and trying to please all your users while maintaining backwards compatibility is -a monumental task. - -With that said, creating something that people use every day to help them make -beautiful software projects is extremely rewarding. Even if your plugin doesn't get -many users, being able to use a tool *you wrote* is very satisfying. - -So if you've got an idea for a plugin that would make Vim better just sit down, learn -about Vimscript, create it, and release it so we can all benefit. - -If you have any questions or comments feel free to hit me up [on -Twitter][@stevelosh]. You might also enjoy following [@dotvimrc][] where I try to -tweet random, bite-sized lines you might like to put in your `.vimrc` file. - -[@stevelosh]: http://twitter.com/stevelosh -[@dotvimrc]: http://twitter.com/dotvimrc - - {% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2011/09/writing-vim-plugins.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2011/09/writing-vim-plugins.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,604 @@ ++++ +title = "Writing Vim Plugins" +snip = "It's pretty much black magic." +date = 2011-09-06T09:13:00Z +draft = false + ++++ + + +A while ago I wrote a [post][vimpost] about switching back to [Vim][]. Since then +I've written two plugins for Vim, one of which has been officially "released". + +A couple of people have asked me if I'd write a guide to creating Vim plugins. +I don't feel confident enough to write an official "guide", but I do have some advice +for Vim plugin authors that might be useful. + +[vimpost]: http://stevelosh.com/blog/2010/09/coming-home-to-vim/ +[Vim]: http://www.vim.org/ + +{{% toc %}} + +Other People Who Know More Than I Do +------------------------------------ + +Writing two decently-sized Vim plugins has given me some experience, but there are +a lot of people that know far more than I do. There are two in particular that come +to mind. I'd love for them to write some guides (or even books) about modern-day Vim +scripting. + +### Tim Pope + +The first is [Tim Pope][]. He's written a ton of Vim plugins like [Pathogen][], +[Surround][], [Repeat][], [Speeddating][] and [Fugitive][]. Each of those is clear, +focused and polished. + +It would be awesome to read a guide on the ins and outs of Vim scripting by him. + +[Tim Pope]: http://tpo.pe/ +[Pathogen]: https://github.com/tpope/vim-pathogen +[Surround]: https://github.com/tpope/vim-surround +[Repeat]: https://github.com/tpope/vim-repeat +[Speeddating]: https://github.com/tpope/vim-speeddating +[Fugitive]: https://github.com/tpope/vim-fugitive + +### Scrooloose + +The other person that comes to mind is [Scrooloose][], author of [NERDTree][], +[NERDCommenter][] and [Syntastic][]. + +His plugins are large and full-featured but work incredibly well, considering how +tricky and painful Vimscript is to work with. I'd love to read a guide on writing +large-scale Vim plugins by him. + +[Scrooloose]: http://got-ravings.blogspot.com/ +[NERDTree]: https://github.com/scrooloose/nerdtree +[NERDCommenter]: https://github.com/scrooloose/nerdcommenter +[Syntastic]: https://github.com/scrooloose/syntastic + +Be Pathogen-Compatible +---------------------- + +It's 2011. When writing your plugin, *please* make its source compatible with +[Pathogen][]. It's very easy to do this — just set up your project's files like +this: + +```text +yourplugin/ + doc/ + yourplugin.txt + plugin/ + yourplugin.vim + ... + README + LICENSE +``` + +This will let users use Pathogen (or [Vundle][]) to install and use your plugin. + +The days of "unzip and drag the files into the right directories" and the horror of +Vimballs are over. Pathogen and Vundle are the right way to manage plugins, so let +your users use them. + +[Vundle]: https://github.com/gmarik/vundle + +Please, For the Love of God, Use normal! +---------------------------------------- + +My first piece of actual scripting advice is something simple but important. If +you're writing a Vim plugin and need to perform some actions, you might be tempted to +use `normal`. Don't. Instead, you need to use `normal!`. + +`normal!` is like `normal`, but ignores mappings the user has set up. If you use +plain old `normal dd` and I've remapped `dd` to do something else, the call will use +my mapping and probably not do what your plugin expects. Using `normal!` ensures +that the call will do what you expect no matter what the user has mapped. + +This is a single instance of a more general theme. Vim is very customizable and users +will do lots of crazy things in their `.vimrc` files. If a key can be mapped or +a setting changed, you *have* to assume that some user of your plugin will have +mapped or changed it. + +Mapping Keys the Right Way +-------------------------- + +Most plugins add key mappings to make them easier to use. Unfortunately this can be +tricky to get right. You can never tell what keys your users have already mapped +themselves, and shadowing someone's favorite key mapping will break their muscle +memory and annoy them to no end. + +### When to Map Keys + +The first question to ask is whether your plugin needs to map keys itself at all. + +My [Gundo][] plugin has only one feature that needs to be mapped to a key in order to +make it useful: the "toggle Gundo" action. + +Gundo doesn't map this key itself, because no matter what "default" mapping I pick +someone will have already mapped it. Instead I added a section right in the README +file that shows how a user can map the key themselves: + +```vim +nnoremap :GundoToggle +``` + +By making users add this line to their `.vimrc` themselves it shows them which key is +used to toggle Gundo (which they would have to know anyway) and also makes it obvious +how to change it to suit their taste. + +[Gundo]: http://sjl.bitbucket.org/gundo.vim/ + +### imap and nmap are Pure Evil + +Sometimes forcing the user to map their own keys won't work. Perhaps your plugin has +many mappings that would be tedious for a user to set up manually (like my +[Threesome][] plugin), or its mappings are mnemonic and wouldn't really make sense if +mapped to other keys. + +I'll talk more about how to deal with this in a moment, but the most important thing +to remember when mapping your own keys is that you must always, *always*, +***always*** use the `noremap` forms of the various `map` commands. + +If you map a key with `nmap` and the user has remapped a key that your mapping uses, +your mapped key will almost certainly not do what you want. Using `nnoremap` will +ignore user mappings and do what you expect. + +This is the same principle as `normal` and `normal!`: *never* trust your users' +configurations. + +[Threesome]: http://sjl.bitbucket.org/threesome.vim + +### Let Me Configure Mappings + +If you feel that your plugin must map some keys, please make those mappings +configurable in some way. + +There are a number of ways to do this. The easiest way is to provide a configuration +option that disables all mappings. The user can them remap the keys as they see fit. +For example: + +```vim +if !exists('g:yourplugin_map_keys') + let g:yourplugin_map_keys = 1 +endif + +if g:yourplugin_map_keys + nnoremap d :call YourPluginDelete() +endif +``` + +Normal users will get the mappings automatically set up for them, and power users can +remap the keys to whatever they wish to avoid shadowing their own mappings. + +If your plugin's mappings all start with a common prefix (like `` or +``) you have another option: allow users to configure this prefix. This +is the approach I've used in [Threesome][]. It works like this: + +```vim +if !exists('g:yourplugin_map_prefix') + let g:yourplugin_map_prefix = '' +endif + +execute "nnoremap" g:yourplugin_map_prefix."d" ":call YourPluginDelete()" +``` + +The `execute` command lets you build the mapping string dynamically so your users can +change the mapping prefix. + +There is a third option for solving this problem: the `hasmapto()` Vim function. +Some plugins will use this to map a command to a key *unless* the user has already +mapped that command to something else. I don't personally like this option because +it feels less clear to me, but I know other people feel differently so I wanted to +mention it. + +Localize Mappings and Settings +------------------------------ + +The next step in being a good Vim plugin author is to try to minimize the effects of +your key mappings and setting changes. Some plugins will need to have global +effects but others will not. + +For example: if you're writing a plugin for working with Python files it should only +take effect for Python buffers, not all buffers. + +### Localizing Mappings + +Key binding are easy to localize to single buffers. All of the `noremap` commands +can take an extra `` argument that will localize the mapping to the current +buffer. + +```vim +" Remaps z globally +nnoremap z :YourPluginFoo + +" Remaps z only in the current buffer +nnoremap z :YourPluginFoo +``` + +However, the problem is that you need to run this command in every buffer you want +the mapping active. To do this your plugin can use an `autocommand`. Here's a full +example, using this concept plus the previously mentioned configuration options: + + if !exists('g:yourplugin_map_keys') + let g:yourplugin_map_keys = 1 + endif + + if !exists('g:yourplugin_map_prefix') + let g:yourplugin_map_prefix = '' + endif + + if g:yourplugin_map_keys + execute "autocommand FileType python" "nnoremap " g:yourplugin_map_prefix."d" ":call YourPluginDelete()" + endif + +Now your plugin will define a key mapping only for Python buffers, and your users can +disable or customize this mapping as they see fit. + +This mapping command is quite ugly. Unfortunately that's the price of using +Vimscript and trying to make a plugin that will work for many users. Later I'll talk +about one possible solution to this ugliness. + +### Localizing Settings + +Just as you should make mappings local to buffers when appropriate, you should do the +same with settings like `foldmethod`, `foldmarker` and `shiftwidth`. Not all +settings can be set locally in a buffer. You can read `:help ` to see +if it's possible. + +You can use `setlocal` instead of `set` to localize settings to individual buffers. +Like with mappings you'll need to use an autocommand to run the `setlocal` command +every time the users opens a new buffer. + +Autoload is Your Friend +----------------------- + +If your plugin is something that users will be using all the time you can skip this +section. + +If you're writing something that will only be used in specific cases, you can help +your users by using Vim's `autoload` functionality to delay loading its code until +the user actually tries to use it. + +The way `autoload` works is fairly simple. Normally you would bind a key to call one of your +plugin's functions with something like this: + +```vim +nnoremap z :call YourPluginFunction() +``` + +You can use autoloading by prepending `yourplugin#` to the name of the function: + +```vim +nnoremap z :call yourplugin#YourPluginFunction() +``` + +When this mapping is run, Vim will do the following: + +1. Check to see if `YourPluginFunction` is already defined. If so, call it. +2. Otherwise, look in `~/.vim/autoload/` for a file named `yourplugin.vim`. +3. If it exists, parse and load the file (which presumably defines + `YourPluginFunction` somewhere inside of it). +4. Call the function. + +This means that instead of putting all of your plugin's code in +`plugin/yourplugin.vim` you can put just the key mapping code there and pull the rest +out into `autoload/yourplugin.vim`. + +If your plugin has a decent amount of code this can reduce the startup time of Vim by +a significant amount. + +Check out the full documentation of `autoload` by running `:help autoload` to learn +much more. + +Backwards Compatibility is a Big Deal +------------------------------------- + +Once you've written your Vim plugin and released it into the wild, you have to +maintain it. Users will find bugs and ask for new features. + +Part of being a responsible developer of any kind, including a Vim plugin author, is +maintaining backwards compatibility, *especially* for tools that users will use every +day and burn into their muscle memory. Users rely on tools to work, and tools that +break backwards compatibility will quickly lose users' trust. + +Maintaining backwards compatibility will cause your plugin's code to get crufty in +spots, but it's the price of maintaining your users' happiness. + +### What Matters for Backards Compatibility? + +For a Vim plugin the most important part of staying backwards compatible is ensuring +that key mappings, customized or not, continue to do what users expect. + +If your plugin maps key `X` to do `Y`, then pressing `X` should *always* do `Y`, even +if you change how `Y` is called by renaming `Y` to `Z`. This may mean changing `Y` +into a wrapper function which simply calls `Z`. + +There are many other aspects of backwards compatibility that you will have to +consider, depending on the purpose of your plugin. The rule of thumb you should +follow is: if a user uses this plugin on a daily basis and has its usage burned into +their muscle memory, updating the plugin should not make them relearn anything. + +### Use Semantic Versioning So I Can Stay Sane + +A fast, simple, easy way to document your plugin's state is to use [semantic +versioning][]. + +Semantic versioning is simply the idea that instead of picking arbitrary version +numbers for releases of your project, you use version numbers that describe the +backwards-compatible state in a meaningful way. + +In a nutshell, these rules describe how you should select version numbers for new +releases: + +* Version numbers have three components: `major.minor.bugfix`. For example: `1.2.4` + or `2.13.0`. +* Versions with a major version of 0 (e.g. `0.2.3`) make no guarantees about + backwards compatibility. You are free to break anything you want. It's only after + you release `1.0.0` that you begin making promises. +* If a release introduces backwards-incompatible changes, increment the major version + number. +* If a release is backwards-compatible, but adds *new* features, increment the minor + version number. +* If a release simply fixes bugs, refactors code, or improves performance, increment + the bugfix version number. + +This simple scheme makes it easy for users to tell (in a broad sense) what has +changed when they update your project. + +If only the bugfix number has changed they can update without fear and continue on +without worrying about changes unless they're curious. + +If the minor version number has changed they might want to look at the changelog to +see what new features they may want to take advantage of, but if they're busy they +can simply update and move on. + +If the major version number has changed it's a major red flag, and they'll want to +read the changelog carefully to see what is different. + +Some people don't like semantic versioning for the following reason: + +> If I have to increment the major version number every time I make +> backwards-incompatible changes, I'll quickly be at ugly versions like 24.1.2! + +To this I say: "Yes, but if that happens you're doing things wrong in the first +place." + +Keep your project in "beta" (i.e. version `0.*.*`) for as long as you need to +experiment freely. *Take your time* and make sure you've gotten things (mostly) +right. Once you release `1.0.0` it's time to start being responsible and caring +about backwards compatibility. + +Breaking functionality all the time harms your users by reducing their productivity +and frustrating them. Yes, it means adding some cruft to your code over time, but +it's the price of not being evil. + +[semantic versioning]: http://semver.org/ + +Document Everything +------------------- + +A critical part of releasing a Vim plugin to the world is writing documentation for +it. Vim has fantastic documentation itself, so your plugins should follow in its +footsteps and provide thorough docs. + +### Pick Some Requirements and Stick to Them + +The most important part of your documentation is telling users what they need to have +in order to use your plugin. Vim runs on nearly every system imaginable and can be +compiled in many different ways, so being specific about your plugin's requirements +will save users a lot of trial and error. + +* Does your plugin only work with Vim version X.Y or later? +* Does it require Python/Ruby/etc support compiled in? Which version? +* Does it not work on Windows? +* Does it rely on an external tool? + +If the answer to any of those questions is "yes", you *must* mention it in the +documentation. + +### Write a README + +The first step to documenting your plugin is to write a README file for the +repository. You can also use the text of this file as the description if you upload +your plugin to the [vim website][], or the content of your plugin's website if you +create one for it. + +Some examples of things to include in your README are: + +* An overview of what the plugin does. +* Screenshots, if possible. +* Requirements. +* Installation instructions. +* Common configuration options that many users will want to know. +* Links to: + * A canonical web address to find the plugin. + * The bug tracker for the plugin. + * The source code or repository of the plugin. + +[vim website]: http://www.vim.org/ + +### Create a Simple Website + +This isn't strictly necessary, but having a simple website for your plugin is an +extra touch that makes it seem more polished. + +It also gives you a canonical URL that people can visit to get the latest information +about your plugin. + +I've made simple sites for both of my plugins: [Gundo][] and [Threesome][]. Feel +free to use them as an example or even take their code and use it for your own plugin +sites if you like. + +### Write a Vim Help Document + +The bulk of your plugin's documentation should be in the form of a Vim help document. +Users are used to using Vim's `:help` and they'll expect to be able to use it to +learn about your plugin. + +Creating a help document is as easy as creating a `doc/yourplugin.txt` file in your +project. It will be indexed automatically by `pathogen#helptags()` so your users +will have the docs at their fingertips. + +Two easy ways to learn the syntax of help files are by reading `:help help-writing` +and using an existing plugin's help file as an example. + +Take your time and craft a beautiful help file you can be proud of. Don't be afraid +to add a bit of personality to your docs to break the dryness. The [syntastic help +file][syntastic help] is a great example (especially the `About` section). + +Things to include in your documentation: + +* A brief overview of the plugin. +* A more in-depth description of how the plugin is used. +* Every single key mapping the plugin creates. +* Ways to extend the plugin, if applicable. +* All configuration variables (including their default values!). +* The plugin's changelog. +* The plugin's license. +* Links to the plugin's repository and bug tracker. + +In a nutshell: your help file should contain *anything* a user would ever need to know +about your plugin. + +[syntastic help]: https://github.com/scrooloose/syntastic/blob/master/doc/syntastic.txt + +### Keep a Changelog + +The last part of documenting your project is keeping a changelog. You can skip this +while your project is still in "beta" (i.e. less than version `1.0.0`) but once you +officially release a real version you need to keep your users informed about what has +changed between releases. + +I like to include this log in the README, the plugin's website, and the +documentation to make it as easy as possible for users to see what's changed. + +Try to keep the language of the changelog at a high enough level for your users to +understand without knowing anything about the implementation of your plugin. Things +like "added feature X" and "fixed bug Y" are great, while things like "refactored the +inner workings of utility function Z" are best left in commit messages. + +Making Vimscript Palatable +-------------------------- + +The worst part about writing Vim plugins is, without a doubt, dealing with Vimscript. +It's an esoteric language that's grown organically over the years seemingly without +any strong design direction. + +Features are added to Vim, then Vimscript features are added to control those +features, then hacky workarounds are added for flexibility. + +The syntax is terse, ugly and inconsistent. Is `" foo` a comment? Sometimes. + +Much of the time you'll spend writing your first plugin will be learning how to do +things in Vimscript. The help documentation on all of its features is thorough, but +it can be hard to find what you're looking for if you don't know the exact name. +Looking through other plugins is often very helpful in pointing you toward what you +need. + +There are a couple of ways to ease the pain of Vimscript, and I'll briefly talk about +two of them here. + +### Wrap. Everything. + +The first piece of advice I have is this: if you want to make your plugins readable +and maintainable then you need to wrap up functionality even more than you would in +other languages. + +For example, my [Gundo][] plugin has a few utility functions that look like this: + +```vim +function! s:GundoGoToWindowForBufferName(name)"{{{ + if bufwinnr(bufnr(a:name)) != -1 + exe bufwinnr(bufnr(a:name)) . "wincmd w" + return 1 + else + return 0 + endif +endfunction"}}} +``` + +This function will go to the window for the given buffer name and gracefully handle +the case where the buffer/window does not exist. It's verbose but much more readable +than the alternative of using that `if` statement in every place I need to switch +windows. + +As you write your plugin you'll "grow" a number of these utility functions. Any time +you duplicate code you should think about creating one, but you should also do so any +time you write a particularly hairy line of Vimscript. Pulling complex lines out +into named functions will save you a lot of reviewing and rethinking down the line. + +### Scripting Vim with Other Languages + +Another option for making Vimscript less painful is to simply not use it much at all. +Vim includes support for creating plugins in a number of other languages like Python +and Ruby. Many plugin authors choose to move nearly all of their code into another +language, using a small Vimscript "wrapper" to expose it to the user. + +I decided to try this approach with [Threesome][] after seeing it used in the +[vim-orgmode][] plugin to great effect. Overall I consider it to be a good idea, +with a few caveats. + +First, using another language will requires your plugin's users to use a version of +Vim compiled with support for that version. In this day and age it's usually not +a problem, but if you want your plugin to run everywhere then it's not an option. + +Using another language adds overhead. You need to not only learn Vimscript but also +the interface between Vim and the language. For small plugins this can add more +complexity to the project than it saves, but for larger plugins it can pay for +itself. It's up to you to decide whether it's worth it. + +Finally, using another language does not entirely insulate you from the +eccentricities of Vimscript. You still need to learn how to do most things in +Vimscript — using another language simply lets you wrap most of this up more neatly +than you otherwise could. + +[vim-orgmode]: https://github.com/jceb/vim-orgmode + +### Unit Testing Will Make You Drink + +Unit testing (and other types of testing) is becoming more and more popular today. +In particular the Python and Ruby communities seem to be getting more and more +excited about it as time goes on. + +Unfortunately, unit testing Vim plugins lies somewhere between "painful" and +"[garden-weasel][]ing your face" on the difficulty scale. + +I tried adding some unit tests to [Gundo][], but even after looking at a number of +frameworks I was spending hours simply trying to get my tests to function. + +I didn't even bother trying to add tests to [Threesome][] because for every hour +I would have spent fighting Vim to create tests I could have cleaned up the code and +fixed bugs instead. + +I'll gladly change my opinion on the subject if someone writes a unit testing +framework for Vim that's as easy to use as [Cram][]. In fact, I'll even buy the +author a $100 bottle of scotch (or whatever they prefer). + +Until that happens I personally don't think it's worth your time to unit test Vim +plugins. Spend your extra hours reading documentation, testing things manually with +a variety of settings, and thinking hard about your code instead. + +[garden-weasel]: http://www.amazon.com/Garden-Weasel-90206/dp/B002ECYRH4 +[Cram]: https://bitheap.org/cram/ + +TL;DR +----- + +Writing Vim plugins is tricky. Vimscript is a rabbit hole of sadness and despair, +and trying to please all your users while maintaining backwards compatibility is +a monumental task. + +With that said, creating something that people use every day to help them make +beautiful software projects is extremely rewarding. Even if your plugin doesn't get +many users, being able to use a tool *you wrote* is very satisfying. + +So if you've got an idea for a plugin that would make Vim better just sit down, learn +about Vimscript, create it, and release it so we can all benefit. + +If you have any questions or comments feel free to hit me up [on +Twitter][@stevelosh]. You might also enjoy following [@dotvimrc][] where I try to +tweet random, bite-sized lines you might like to put in your `.vimrc` file. + +[@stevelosh]: http://twitter.com/stevelosh +[@dotvimrc]: http://twitter.com/dotvimrc + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/04/volatile-software.html --- a/content/blog/2012/04/volatile-software.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,323 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "Volatile Software" - snip: "Our culture is one of pain and suffering." - created: 2012-04-23 14:00:00 - %} - -{% block article %} - -The following is the text of an email I sent to [The Listserve][], which was -sent to that list on April 22, 2012. - -[The Listserve]: http://thelistserve.com/ - -
- -I want to use my fifteen minutes of fame on The Listserve to rant about -something that's close to my heart: the stability of the software I use. - -NOTE: This is written for people who create software. If you don't do that you -probably won't find this very interesting. Sorry! Maybe you could read Text -from Dog if you haven't seen it already? Either way, have a nice -morning/afternoon/evening! - -The Situation -------------- - -Every time I get a new computer, I go through the same song and dance: - -1. Look at what programs and packages I have installed on the old computer. -2. Install these programs on the new computer. -3. Copy over my configuration files from the old computer to the new one. -4. Spend the rest of my day fixing all the things that broke because I'm using - a newer version of program X. - -Step 4 is always the most painful part of getting a new machine. Always. - -Without fail I spend several hours tweaking configuration files, adjusting my -workflow, and so on because I've upgraded to a new version of foo which doesn't -support option X any more or requires library Y version N+1 now. - -Getting a new computer should be a *pleasant* experience! The unboxing from the -sleek packaging, that "new laptop" smell, the nostalgia of the default desktop -image. Why does this horrible step 4 have to exist and how can we get rid of -it? - -The Divide ----------- - -I've noticed something interesting lately: I can categorize almost *all* of the -software I use into two distinct groups: - -* Software that breaks pretty much *every* time I update it (e.g. weechat, - offlineimap, Clojure, many Python packages, Skype). -* Software that almost *never* breaks when I update it (e.g. Mercurial, git, - tmux, Python, ack, zsh, Vim, Dropbox). - -Software that falls in between these two extremes is surprisingly rare. There -seems to be a pretty clean divide between the two groups. - -This makes me think that there's some special attribute or quality of the -second group (or its authors) which the first one lacks. - -Brokenness ----------- - -I think it's important that I nail down what I mean by "breaks" or "is broken". -I don't necessarily just mean the introduction of "new bugs". - -When I say that a program "breaks", I mean: - -* When I update from version X to version Y of a program, library, or language... -* Without changing my configuration files, source code, etc... -* The resulting combination doesn't work properly - -In effect, I'm saying that "breaking backwards compatibility" means "the program -is broken"! - -This may be a strong statement, but I stand by it in most cases. - -Backwards compatibility matters! Every time someone makes a backwards -incompatible change in a program or library, they cost the world the following -amount of time: - - Number of people Time it takes each person - using that part of X to figure out what changed - the program and how to fix it - -Often this can be a significant amount of time! - -The Process of Updating ------------------------ - -When pointing out a backwards incompatible change to someone, you'll often get -a response similar to this: - -> "Well, I mentioned that backwards incompatibility in the changelog, so what -> the hell, man!" - -This is not a satisfactory answer. - -When I'm updating a piece of software there's a good chance it's not because I'm -specifically updating *that program*. I might be: - -* Moving to a new computer. -* Running a "$PACKAGE\_MANAGER update" command. -* Moving a website to a bigger VPS and reinstalling all the libraries. - -In those cases (and many others) I'm not reading the release notes for -a specific program or library. I'm not going to find out about the brokenness -until I try to use the program the next time. - -If I'm lucky the program will have a "this feature is now deprecated, read the -docs" error message. That's still a pain, but at least it's less confusing than -just getting a traceback, or worst of all: silently changing the behavior of -a feature. - -Progress --------- - -I completely understand that when moving *backwards* to an older version -I should expect problems. The older version hasn't had the benefit of the extra -work done on the new version, so of course it should be less stable. - -But when I'm *updating* to a higher version number the software should be -*better* and *more stable*! It has had more work done on it, and I assume no -one is actively trying to make software worse, so why does something that -previously worked no longer work? - -We're supposed to be making *progress* as we move forward. The software has had -*more* work done on it, why does it not function correctly *now* when it -functioned correctly *before*? - -Yes, this means developers will need to add extra code to handle old -input/configuration. Yes, this is a pain in the ass, but *the entire point of -most software is to save people time by automating things*. Again, every -backwards-incompatible change costs the world an amount of time: - - Number of people Time it takes each person - using that part of X to figure out what changed - the program and how to fix it - -If our goal is to *save time* then we should not make changes that *cost time*. -Or at least we should not make such changes lightly. - -A Culture of Sadness --------------------- - -Proof that this is a real issue can be found in the tools we use every day. As -programmers we've invented elaborate dependency systems to deal with it. - -We say `pip install django==1.3` or put `[clojure "1.2"]` in our Leiningen -project.clj files to avoid using the newest versions because they'll break. - -Step back and look at this for a second. - -What the hell? - -What the *hell*? - -We have invented software with features designed to help us use *old* versions -of other software! - -We have *written code* to *avoid* using the "latest and greatest" software! - -Obviously this is not entirely bad, but the fact that manually specifying -version numbers to avoid running *newer* code is commonplace, expected, and -a "best practice" horrifies me. - -I would *love* to be able to say something like this in my requirements.txt and -project.clj files: - -> Of *course* I want the latest version of library X! I want *all* the newest -> bug fixes and improvements! - -Unfortunately I can't do that right now because so many projects make backwards -incompatible changes all the time. - -The moment I try to build the project at some point in the future I'll be sent -on a wild goose chase to figure out what function moved into what other -namespace and what other function was split into its own library and dammit the -documentation on the project's website is autogenerated from the tip of its git -repo and so it doesn't apply to the latest actual version and jesus christ -I think I'll just quit programming and teach dance full time instead even though -I'll go hungry. - -The Tradeoff ------------- - -One could argue that sometimes backwards incompatible changes cost time up front -but save time in the long run by making the software more "elegant" and "lean". - -While I'm sure there are cases where this is true, I feel like it's a cop out -most of the time. Allow me to illustrate this with a helpful Venn diagram: - - :::text - ------- - /3333333\ - |333333333| - \3333333/ - ------- - - 11 -> People who give a shit what a program's codebase - 11 looks like. - - 22 -> The authors of said program. - 22 - -For libraries where the author is the only user, none of this rant applies. -You're free! Break as much as you like! - -For the majority of libraries, however, there are probably vastly more "users" -than "authors". Saving a few hours of the authors' own time has to be weighed -against the 10 minutes each that the hundreds of users will have to spend -figuring out what happened and working around it. - -I want to be clear: being backwards compatible *doesn't* mean sacrificing new -features! New features can still be added! Refactoring can still happen! - -In most cases keeping backwards compatibility simply means maintaining a bit of -wrapper code to support people using the previous version. - -For example: in Python, if we moved the public foo() function to a new module, -we'd put the following line in the original module: - - :::python - from newmodule import new_foo as foo - -Is it pretty? Hell no! But this single line of code will probably save more -people more time than most of the other lines in the project! - -This may just be an artifact of how my brain is wired, but I actually get -a sense of satisfaction from writing code that bridges the gap between older -versions and new. - -I can almost hear a little voice in my head saying: - -> "Mwahaha, I'll slip this refactoring past them and they'll never even know it -> happened!" - -Maybe it's just me, but I think that "glue" code can be clever and beautiful in -its own right. - -It may not bring a smile to anyone's face like a shiny new feature, but it -prevents many frowns instead, and preventing a frown makes the world a happier -place just as much as creating a smile! - -Exceptions ----------- - -One case where I feel the backwards incompatibility tradeoff *is* worth it is -security. - -A good example of this is Django's change which made AJAX requests no longer be -exempt from CSRF checks. It was backwards incompatible and I'm sure it broke -some people's projects, but I think it was the right thing to do because it -improved security. - -I also think it's unreasonable to expect all software to be perfectly ready from -its first day. - -Sometimes software needs to get poked and prodded in the real world before it's -fully baked, and until then requiring strict backwards compatibility will do -more harm than good. - -By all means, backwards compatibility should be thrown to the wind in the first -stage of a project's life. At the beginning it needs to find its legs, like -a baby gazelle on the Serengeti. But at some point the project needs to get its -balance, grow up, and start concerning itself with backwards compatibility. - -But when should that happen? - -A Solution ----------- - -I think there's a simple, intuitive way to mark the transition of a piece of -software from "volatile" to "stable": - -**Version 1.0** - -Before version 1, software can change and evolve rapidly with no regards for -breaking, but once that first number becomes "greater than or equal to 1" it's -time to be a responsible member of the software community and start thinking -about the real humans whose time gets wasted for every breaking change. - -This is the approach semantic versioning takes, and I think it's the right one. - -I know a lot of people dislike semantic versioning. They hate how requires -incrementing the major version number every time a breaking change is made. - -I consider it to be a *good* thing. - -You *should* pause and carefully consider making a change that will break -people's current code. - -You *should* be ashamed if your project is at version 43.0.0 because you've made -42 breaking changes. That's 43 times you've disregarded your users' time! -That's a bad thing! - -As programmers we need to start caring about the people we write software for. - -Before making a change that's going to cause other people pain, we should ask -ourselves if it's really worth the cost. Sometimes it is, but many times it's -not, and we can wrap the change up so it doesn't hurt anyone. - -So please, before you make that backwards incompatible change, think of the -other human beings who are going to smack their monitors when your software -breaks. - -Further Reading ---------------- - -I'm certainly not the only person to notice this problem. Many smarter people -than me have talked about it. If you want to read more you might want to look -up some or all of the following (Google is your friend): - -* The Semantic Versioning spec (the specific numbering details don't matter as - much as the philosophy). -* Anything Matt Mackall has written on the Mercurial mailing list (especially - the mails where he sounds especially grouchy). -* Anything about "software rot" or "code rot". -{% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/04/volatile-software.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2012/04/volatile-software.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,322 @@ ++++ +title = "Volatile Software" +snip = "Our culture is one of pain and suffering." +date = 2012-04-23T14:00:00Z +draft = false + ++++ + +The following is the text of an email I sent to [The Listserve][], which was +sent to that list on April 22, 2012. + +[The Listserve]: http://thelistserve.com/ + +
+ +I want to use my fifteen minutes of fame on The Listserve to rant about +something that's close to my heart: the stability of the software I use. + +NOTE: This is written for people who create software. If you don't do that you +probably won't find this very interesting. Sorry! Maybe you could read Text +from Dog if you haven't seen it already? Either way, have a nice +morning/afternoon/evening! + +The Situation +------------- + +Every time I get a new computer, I go through the same song and dance: + +1. Look at what programs and packages I have installed on the old computer. +2. Install these programs on the new computer. +3. Copy over my configuration files from the old computer to the new one. +4. Spend the rest of my day fixing all the things that broke because I'm using + a newer version of program X. + +Step 4 is always the most painful part of getting a new machine. Always. + +Without fail I spend several hours tweaking configuration files, adjusting my +workflow, and so on because I've upgraded to a new version of foo which doesn't +support option X any more or requires library Y version N+1 now. + +Getting a new computer should be a *pleasant* experience! The unboxing from the +sleek packaging, that "new laptop" smell, the nostalgia of the default desktop +image. Why does this horrible step 4 have to exist and how can we get rid of +it? + +The Divide +---------- + +I've noticed something interesting lately: I can categorize almost *all* of the +software I use into two distinct groups: + +* Software that breaks pretty much *every* time I update it (e.g. weechat, + offlineimap, Clojure, many Python packages, Skype). +* Software that almost *never* breaks when I update it (e.g. Mercurial, git, + tmux, Python, ack, zsh, Vim, Dropbox). + +Software that falls in between these two extremes is surprisingly rare. There +seems to be a pretty clean divide between the two groups. + +This makes me think that there's some special attribute or quality of the +second group (or its authors) which the first one lacks. + +Brokenness +---------- + +I think it's important that I nail down what I mean by "breaks" or "is broken". +I don't necessarily just mean the introduction of "new bugs". + +When I say that a program "breaks", I mean: + +* When I update from version X to version Y of a program, library, or language... +* Without changing my configuration files, source code, etc... +* The resulting combination doesn't work properly + +In effect, I'm saying that "breaking backwards compatibility" means "the program +is broken"! + +This may be a strong statement, but I stand by it in most cases. + +Backwards compatibility matters! Every time someone makes a backwards +incompatible change in a program or library, they cost the world the following +amount of time: + + Number of people Time it takes each person + using that part of X to figure out what changed + the program and how to fix it + +Often this can be a significant amount of time! + +The Process of Updating +----------------------- + +When pointing out a backwards incompatible change to someone, you'll often get +a response similar to this: + +> "Well, I mentioned that backwards incompatibility in the changelog, so what +> the hell, man!" + +This is not a satisfactory answer. + +When I'm updating a piece of software there's a good chance it's not because I'm +specifically updating *that program*. I might be: + +* Moving to a new computer. +* Running a "$PACKAGE\_MANAGER update" command. +* Moving a website to a bigger VPS and reinstalling all the libraries. + +In those cases (and many others) I'm not reading the release notes for +a specific program or library. I'm not going to find out about the brokenness +until I try to use the program the next time. + +If I'm lucky the program will have a "this feature is now deprecated, read the +docs" error message. That's still a pain, but at least it's less confusing than +just getting a traceback, or worst of all: silently changing the behavior of +a feature. + +Progress +-------- + +I completely understand that when moving *backwards* to an older version +I should expect problems. The older version hasn't had the benefit of the extra +work done on the new version, so of course it should be less stable. + +But when I'm *updating* to a higher version number the software should be +*better* and *more stable*! It has had more work done on it, and I assume no +one is actively trying to make software worse, so why does something that +previously worked no longer work? + +We're supposed to be making *progress* as we move forward. The software has had +*more* work done on it, why does it not function correctly *now* when it +functioned correctly *before*? + +Yes, this means developers will need to add extra code to handle old +input/configuration. Yes, this is a pain in the ass, but *the entire point of +most software is to save people time by automating things*. Again, every +backwards-incompatible change costs the world an amount of time: + + Number of people Time it takes each person + using that part of X to figure out what changed + the program and how to fix it + +If our goal is to *save time* then we should not make changes that *cost time*. +Or at least we should not make such changes lightly. + +A Culture of Sadness +-------------------- + +Proof that this is a real issue can be found in the tools we use every day. As +programmers we've invented elaborate dependency systems to deal with it. + +We say `pip install django==1.3` or put `[clojure "1.2"]` in our Leiningen +project.clj files to avoid using the newest versions because they'll break. + +Step back and look at this for a second. + +What the hell? + +What the *hell*? + +We have invented software with features designed to help us use *old* versions +of other software! + +We have *written code* to *avoid* using the "latest and greatest" software! + +Obviously this is not entirely bad, but the fact that manually specifying +version numbers to avoid running *newer* code is commonplace, expected, and +a "best practice" horrifies me. + +I would *love* to be able to say something like this in my requirements.txt and +project.clj files: + +> Of *course* I want the latest version of library X! I want *all* the newest +> bug fixes and improvements! + +Unfortunately I can't do that right now because so many projects make backwards +incompatible changes all the time. + +The moment I try to build the project at some point in the future I'll be sent +on a wild goose chase to figure out what function moved into what other +namespace and what other function was split into its own library and dammit the +documentation on the project's website is autogenerated from the tip of its git +repo and so it doesn't apply to the latest actual version and jesus christ +I think I'll just quit programming and teach dance full time instead even though +I'll go hungry. + +The Tradeoff +------------ + +One could argue that sometimes backwards incompatible changes cost time up front +but save time in the long run by making the software more "elegant" and "lean". + +While I'm sure there are cases where this is true, I feel like it's a cop out +most of the time. Allow me to illustrate this with a helpful Venn diagram: + +```text + ------- + /3333333\ + |333333333| + \3333333/ + ------- + +11 -> People who give a shit what a program's codebase +11 looks like. + +22 -> The authors of said program. +22 +``` + +For libraries where the author is the only user, none of this rant applies. +You're free! Break as much as you like! + +For the majority of libraries, however, there are probably vastly more "users" +than "authors". Saving a few hours of the authors' own time has to be weighed +against the 10 minutes each that the hundreds of users will have to spend +figuring out what happened and working around it. + +I want to be clear: being backwards compatible *doesn't* mean sacrificing new +features! New features can still be added! Refactoring can still happen! + +In most cases keeping backwards compatibility simply means maintaining a bit of +wrapper code to support people using the previous version. + +For example: in Python, if we moved the public foo() function to a new module, +we'd put the following line in the original module: + +```python +from newmodule import new_foo as foo +``` + +Is it pretty? Hell no! But this single line of code will probably save more +people more time than most of the other lines in the project! + +This may just be an artifact of how my brain is wired, but I actually get +a sense of satisfaction from writing code that bridges the gap between older +versions and new. + +I can almost hear a little voice in my head saying: + +> "Mwahaha, I'll slip this refactoring past them and they'll never even know it +> happened!" + +Maybe it's just me, but I think that "glue" code can be clever and beautiful in +its own right. + +It may not bring a smile to anyone's face like a shiny new feature, but it +prevents many frowns instead, and preventing a frown makes the world a happier +place just as much as creating a smile! + +Exceptions +---------- + +One case where I feel the backwards incompatibility tradeoff *is* worth it is +security. + +A good example of this is Django's change which made AJAX requests no longer be +exempt from CSRF checks. It was backwards incompatible and I'm sure it broke +some people's projects, but I think it was the right thing to do because it +improved security. + +I also think it's unreasonable to expect all software to be perfectly ready from +its first day. + +Sometimes software needs to get poked and prodded in the real world before it's +fully baked, and until then requiring strict backwards compatibility will do +more harm than good. + +By all means, backwards compatibility should be thrown to the wind in the first +stage of a project's life. At the beginning it needs to find its legs, like +a baby gazelle on the Serengeti. But at some point the project needs to get its +balance, grow up, and start concerning itself with backwards compatibility. + +But when should that happen? + +A Solution +---------- + +I think there's a simple, intuitive way to mark the transition of a piece of +software from "volatile" to "stable": + +**Version 1.0** + +Before version 1, software can change and evolve rapidly with no regards for +breaking, but once that first number becomes "greater than or equal to 1" it's +time to be a responsible member of the software community and start thinking +about the real humans whose time gets wasted for every breaking change. + +This is the approach semantic versioning takes, and I think it's the right one. + +I know a lot of people dislike semantic versioning. They hate how requires +incrementing the major version number every time a breaking change is made. + +I consider it to be a *good* thing. + +You *should* pause and carefully consider making a change that will break +people's current code. + +You *should* be ashamed if your project is at version 43.0.0 because you've made +42 breaking changes. That's 43 times you've disregarded your users' time! +That's a bad thing! + +As programmers we need to start caring about the people we write software for. + +Before making a change that's going to cause other people pain, we should ask +ourselves if it's really worth the cost. Sometimes it is, but many times it's +not, and we can wrap the change up so it doesn't hurt anyone. + +So please, before you make that backwards incompatible change, think of the +other human beings who are going to smack their monitors when your software +breaks. + +Further Reading +--------------- + +I'm certainly not the only person to notice this problem. Many smarter people +than me have talked about it. If you want to read more you might want to look +up some or all of the following (Google is your friend): + +* The Semantic Versioning spec (the specific numbering details don't matter as + much as the philosophy). +* Anything Matt Mackall has written on the Mercurial mailing list (especially + the mails where he sounds especially grouchy). +* Anything about "software rot" or "code rot". diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/07/caves-of-clojure-01.html --- a/content/blog/2012/07/caves-of-clojure-01.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,165 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "The Caves of Clojure: Part 1" - snip: "Getting a Roguelike up and running." - created: 2012-07-07 17:00:00 - %} - -{% block article %} - -Lately I've had an urge to start playing a few games again, namely [Nethack][] -and [Dwarf Fortress][] (the latter being triggered by [this book][df-book]). -Aside from being incredibly fun, they also made me want to play around with -writing a [roguelike][] game of my own. - -[Nethack]: http://www.nethack.org/ -[Dwarf Fortress]: http://www.bay12games.com/dwarves/ -[df-book]: http://www.amazon.com/gp/product/1449314945/ref=as_li_ss_tl?ie=UTF8&camp=1789&creative=390957&creativeASIN=1449314945&linkCode=as2&tag=stelos-20 -[roguelike]: https://en.wikipedia.org/wiki/Roguelike - -I could write them in Python, but lately I've been falling out of love with the -language. It's a solid workhorse that's not exciting or beautiful to me at all -any more. My affection has shifted more toward Clojure. Despite its Rubyesque -culture of brokenness, rampant lack of documentation, and warty JVM interop it's -still a wonderful language that's captured me (for now). - -I've had the idea of writing a few roguelike games for a while, but the other -day I stumbled on [Trystan Spangler's blog][trystan] and his [series of -articles][trystan-tut] that walk through writing a roguelike game in Java. - -[trystan]: http://trystans.blogspot.com/ -[trystan-tut]: http://trystans.blogspot.com/2011/08/roguelike-tutorial-01-java-eclipse.html - -I'm going to do a series of blog posts, each corresponding roughly to one of -Trystan's posts, as I work my way through writing a roguelike in Clojure. I may -or may not get all of the way through his twenty-post series. We'll see. - -I'll assume you know Clojure during this series and won't be explaining every -single thing. - -If you want to follow along, the code for this series is [on Bitbucket][bb] and -[on GitHub][gh]. There are tags like `entry-01` in the repo which you can -update to and see the code as it stood at the end of that entry. - -[bb]: http://bitbucket.org/sjl/caves/ -[gh]: http://github.com/sjl/caves/ - -Let's jump in. This entry corresponds to [post one in Trystan's -tutorial][trystan-tut]. - -[TOC] - -Summary -------- - -The first thing to do is to bootstrap an environment for development. I'll be -using Clojure 1.4, Leiningen 2, and [clojure-lanterna][] 0.9.0. - -Trystan used Eclipse but I'll be using Vim and my fork of SLIMV. - -[clojure-lanterna]: http://sjl.bitbucket.org/clojure-lanterna/ - -project.clj ------------ - -I'm starting with a fairly simple `project.clj` file: - - :::clojure - (defproject caves "0.1.0-SNAPSHOT" - :description "The Caves of Clojure" - :url "http://stevelosh.com/blog/2012/07/caves-of-clojure-01/" - :license {:name "MIT/X11"} - :dependencies [[org.clojure/clojure "1.4.0"] - [clojure-lanterna "0.9.0"]] - :main caves.core) - -Nothing particularly crazy here. - -clojure-lanterna ----------------- - -Trystan used his own library called AsciiPanel to handle the drawing of -characters to the screen. I'm going to use my own library [clojure-lanterna][], -which is a wrapper around the [Lanterna][] Java library. - -I chose Lanterna because it supports drawing to a Swing-based "terminal" and to -the actual console. But unlike some other libraries, it doesn't actually link -against native code (it's pure Java) so it's more portable. - -Being able to output to either a console or a GUI means that I can develop -through Swank really easily with the Swing terminal, but actuall play the -finished product in a terminal like God intended. - -[Lanterna]: https://code.google.com/p/lanterna/ - -core.clj --------- - -The full `core.clj` file I came up with was this: - - :::clojure - (ns caves.core - (:require [lanterna.screen :as s])) - - - (defn main [screen-type] - (let [screen (s/get-screen screen-type)] - (s/in-screen screen - (s/put-string screen 0 0 "Welcome to the Caves of Clojure!") - (s/put-string screen 0 1 "Press any key to exit...") - (s/redraw screen) - (s/get-key-blocking screen)))) - - - (defn -main [& args] - (let [args (set args) - screen-type (cond - (args ":swing") :swing - (args ":text") :text - :else :auto)] - (main screen-type))) - -We're very much just bootstrapping things for now. The `-main` function parses -the command line arguments to figure out what type of terminal we should use, -and then passes that along to `main`. - -For now, `main` simply: - -* Gets an appropriately-typed terminal. -* Outputs a simple message. -* Redraws the screen (clojure-lanterna's screen layer buffers output until you - tell it to redraw). -* Waits for the user to press a key. -* Exits. - -I chose to split the meat of the setup into a `main` function instead of just -putting everything in `-main` because that will make it easy for me to work -through Swank instead of the command line as we'll see in a moment. - -Running -------- - -There are two main ways to actually make this thing work. - -First, we can run it as a standalone program with `lein trampoline run` at the -command line. I can pass a parameter to specify what type of terminal to use, -like `lein trampoline run :swing`. - -We can also run it from a REPL, or a SLIME/Swank environment by simply -evaluating `(main :swing)` in the namespace. This is why I split out everything -but the command line argument parsing into a separate function. - -Either way, once you run it you get something like this: - -![Screenshot](/media/images{{ parent_url }}/caves-01-01.png) - -This is the Swing terminal which I happened to start from swank. Press a key -and it will go away. - -It doesn't look like much, but it's a [black triangle][]. In the next entry -we'll start doing more interesting things. - -[black triangle]: http://rampantgames.com/blog/2004/10/black-triangle.html - -{% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/07/caves-of-clojure-01.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2012/07/caves-of-clojure-01.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,164 @@ ++++ +title = "The Caves of Clojure: Part 1" +snip = "Getting a Roguelike up and running." +date = 2012-07-07T17:00:00Z +draft = false + ++++ + +Lately I've had an urge to start playing a few games again, namely [Nethack][] +and [Dwarf Fortress][] (the latter being triggered by [this book][df-book]). +Aside from being incredibly fun, they also made me want to play around with +writing a [roguelike][] game of my own. + +[Nethack]: http://www.nethack.org/ +[Dwarf Fortress]: http://www.bay12games.com/dwarves/ +[df-book]: http://www.amazon.com/gp/product/1449314945/ref=as_li_ss_tl?ie=UTF8&camp=1789&creative=390957&creativeASIN=1449314945&linkCode=as2&tag=stelos-20 +[roguelike]: https://en.wikipedia.org/wiki/Roguelike + +I could write them in Python, but lately I've been falling out of love with the +language. It's a solid workhorse that's not exciting or beautiful to me at all +any more. My affection has shifted more toward Clojure. Despite its Rubyesque +culture of brokenness, rampant lack of documentation, and warty JVM interop it's +still a wonderful language that's captured me (for now). + +I've had the idea of writing a few roguelike games for a while, but the other +day I stumbled on [Trystan Spangler's blog][trystan] and his [series of +articles][trystan-tut] that walk through writing a roguelike game in Java. + +[trystan]: http://trystans.blogspot.com/ +[trystan-tut]: http://trystans.blogspot.com/2011/08/roguelike-tutorial-01-java-eclipse.html + +I'm going to do a series of blog posts, each corresponding roughly to one of +Trystan's posts, as I work my way through writing a roguelike in Clojure. I may +or may not get all of the way through his twenty-post series. We'll see. + +I'll assume you know Clojure during this series and won't be explaining every +single thing. + +If you want to follow along, the code for this series is [on Bitbucket][bb] and +[on GitHub][gh]. There are tags like `entry-01` in the repo which you can +update to and see the code as it stood at the end of that entry. + +[bb]: http://bitbucket.org/sjl/caves/ +[gh]: http://github.com/sjl/caves/ + +Let's jump in. This entry corresponds to [post one in Trystan's +tutorial][trystan-tut]. + +{{% toc %}} + +Summary +------- + +The first thing to do is to bootstrap an environment for development. I'll be +using Clojure 1.4, Leiningen 2, and [clojure-lanterna][] 0.9.0. + +Trystan used Eclipse but I'll be using Vim and my fork of SLIMV. + +[clojure-lanterna]: http://sjl.bitbucket.org/clojure-lanterna/ + +project.clj +----------- + +I'm starting with a fairly simple `project.clj` file: + +```clojure +(defproject caves "0.1.0-SNAPSHOT" + :description "The Caves of Clojure" + :url "http://stevelosh.com/blog/2012/07/caves-of-clojure-01/" + :license {:name "MIT/X11"} + :dependencies [[org.clojure/clojure "1.4.0"] + [clojure-lanterna "0.9.0"]] + :main caves.core) +``` + +Nothing particularly crazy here. + +clojure-lanterna +---------------- + +Trystan used his own library called AsciiPanel to handle the drawing of +characters to the screen. I'm going to use my own library [clojure-lanterna][], +which is a wrapper around the [Lanterna][] Java library. + +I chose Lanterna because it supports drawing to a Swing-based "terminal" and to +the actual console. But unlike some other libraries, it doesn't actually link +against native code (it's pure Java) so it's more portable. + +Being able to output to either a console or a GUI means that I can develop +through Swank really easily with the Swing terminal, but actuall play the +finished product in a terminal like God intended. + +[Lanterna]: https://code.google.com/p/lanterna/ + +core.clj +-------- + +The full `core.clj` file I came up with was this: + +```clojure +(ns caves.core + (:require [lanterna.screen :as s])) + + +(defn main [screen-type] + (let [screen (s/get-screen screen-type)] + (s/in-screen screen + (s/put-string screen 0 0 "Welcome to the Caves of Clojure!") + (s/put-string screen 0 1 "Press any key to exit...") + (s/redraw screen) + (s/get-key-blocking screen)))) + + +(defn -main [& args] + (let [args (set args) + screen-type (cond + (args ":swing") :swing + (args ":text") :text + :else :auto)] + (main screen-type))) +``` + +We're very much just bootstrapping things for now. The `-main` function parses +the command line arguments to figure out what type of terminal we should use, +and then passes that along to `main`. + +For now, `main` simply: + +* Gets an appropriately-typed terminal. +* Outputs a simple message. +* Redraws the screen (clojure-lanterna's screen layer buffers output until you + tell it to redraw). +* Waits for the user to press a key. +* Exits. + +I chose to split the meat of the setup into a `main` function instead of just +putting everything in `-main` because that will make it easy for me to work +through Swank instead of the command line as we'll see in a moment. + +Running +------- + +There are two main ways to actually make this thing work. + +First, we can run it as a standalone program with `lein trampoline run` at the +command line. I can pass a parameter to specify what type of terminal to use, +like `lein trampoline run :swing`. + +We can also run it from a REPL, or a SLIME/Swank environment by simply +evaluating `(main :swing)` in the namespace. This is why I split out everything +but the command line argument parsing into a separate function. + +Either way, once you run it you get something like this: + +![Screenshot](/media/images/blog/2012/07/caves-01-01.png) + +This is the Swing terminal which I happened to start from swank. Press a key +and it will go away. + +It doesn't look like much, but it's a [black triangle][]. In the next entry +we'll start doing more interesting things. + +[black triangle]: http://rampantgames.com/blog/2004/10/black-triangle.html + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/07/caves-of-clojure-02.html --- a/content/blog/2012/07/caves-of-clojure-02.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,581 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "The Caves of Clojure: Part 2" - snip: "Dealing with state." - created: 2012-07-08 9:26:00 - %} - -{% block article %} - -This post is part of an ongoing series. If you haven't already done so, you -should probably start at [the beginning][]. - -This entry corresponds to [post two in Trystan's tutorial][trystan-tut]. - -If you want to follow along, the code for the series is [on Bitbucket][bb] and -[on GitHub][gh]. Update to the `entry-02` tag to see the code as it stands -after this post. - -[the beginning]: /blog/2012/07/caves-of-clojure-01/ -[trystan-tut]: http://trystans.blogspot.com/2011/08/roguelike-tutorial-02-input-output.html -[bb]: http://bitbucket.org/sjl/caves/ -[gh]: http://github.com/sjl/caves/ - -[TOC] - -Summary -------- - -In Trystan's second post he introduces the concept of the game loop, as well as -what he calls "screens": objects that handle drawing the interface and -processing user input. - -I could try to port his design directly over to Clojure, but instead I wanted to -step back and see if I could find a way to make things more functional. - -I think I've figured out a way to make it work, so I'm going to implement that. - -State ------ - -When I first started thinking about how to model the game's state and the main -game loop I had lots of crazy ideas bouncing around in my head. Most of them -involved an immutable world (immutable! good!) and agents representing Trystan's -"screens". - -The more I thought about it, though, the more it looked like the agents would -wind up being a tangled mess. I put down the keyboard, took a shower, had -dinner with a friend, and let the problem roll around in my head for a bit. - -At some point the following train of thought happened somewhere in my brain: - -* The immutable "state" that I keep should contain *everything* needed to render - the game on the user's screen. -* I originally thought I'd need to track the "world" as the state, but the world - isn't enough! -* In addition to the world, the user interface (menus, stats, etc) is also - rendered. - -So instead of keeping a "world" as my state, I'm going to keep a "game". - -The User Interface ------------------- - -If I'm going to keep track of the user interface in the "game" state, I need -a way to represent it. - -There are two halves to the user interface: "input" and "output". First let's -consider output. - -Trystan's screens are objects that handle their own drawing. At any given time -there's one "active" screen object, which gets asked to draw itself. If you -peek ahead in his tutorial you'll see that he ends up introducing a "subscreen" -concept to get screens layered on top of each other. - -Instead of having a single active screen with subscreens, I decided to keep -a flat vector of screens. The last screen in the vector is the "active" one, -and is effectively a subscreen of the one that comes before it. - -At this point I'm going to switch terms. Unfortunately Lanterna uses the word -"screen" to mean something and I didn't want to try to keep two separate -concepts under the same word, so in the code I called screens "UIs", and from -now on I'll be using that word. - -So what *is* a UI in the code? Well, it's basically just a map with a `:kind` -mapping specifying what kind of UI it is! It also might have some extra keys -and values to represent its state too. - -For example, at some point I expect to have a UI stack that looks something like -this: - - :::clojure - [{:kind :play} - {:kind :throw} - {:kind :inventory-select}] - -This would be the UI stack when you were throwing something in your inventory at -a monster and were choosing what to throw. Once you pick an item you'd need to -target a monster, so the stack would become: - - :::clojure - [{:kind :play} - {:kind :throw :item foo} - {:kind :target}] - -Now the last (i.e.: "active") UI is the targetting UI, but also notice that the -throwing UI has a bit of state attached to it now (the item the user picked). -We'll talk about how that got there a bit later. - -As I said before, the "state" for our game is going to be a "game", which -consists of the world and the user interface. So our "game" object is going to -be a map that looks something like this: - - :::clojure - {:world {} - :uis [,,,]} - -For now the `:world` is empty. In the future it'll contain stuff like the tiles -of the map, the monsters, the player, and lots of other stuff. The `:uis` is -the UI stack. - -Between the two I have enough information to draw the game fully to the user's -terminal by doing something like: `(map #(draw-ui ui game) (:uis game))`. We'll -see the real code shortly, but that's actually pretty close. - -User Input ----------- - -In an imperative programming style our game loop would look something like this: - -1. Draw the screen. -2. Get some input from the user. -3. Process that input, modifying the world. -4. GOTO 1. - -In this functional loop, I want it to look more like this: - -1. Draw the screen. -2. Get some user input. -3. Process the input and the game to get a new game. -4. Recur with this new game. - -How do I handle user input? Well it depends on the current UI -- pressing `d` -at the main screen will do something different than pressing it in an inventory -selection screen, for example. - -So the UIs need to know how to handle input. There are a number of different -ways I can do that. One option might be to have `:handle-input (fn ...)` as -part of the UI. I chose a different route which you'll see below, but that's -not important for now. - -The important part is that one I glossed over in the last section. How do I go -from this: - - :::clojure - [{:kind :play} - {:kind :throw} - {:kind :inventory-select}] - -to this: - - :::clojure - [{:kind :play} - {:kind :throw :item foo} - {:kind :target}] - -Let's follow the proposed game loop and see what happens. - -1. Draw the play UI, then the throw UI, then the inventory UI. -2. Get a keypress from the user. -3. Give that keypress and the game itself to the UI input handling function to - get a new game. -4. Recur with this new game. - -Step three is the tricky part. What does the inventory handler need to do to -give back a new game? - -It would need to pop itself off the UI stack (which is okay), put the selected -item in the previous UI (a bit scary, but probably not a problem in practice), -and create the targeting UI. - -This last part is a deal breaker. The inventory selection UI shouldn't know -anything about the targeting UI, because then I won't be able to reuse it for -other functions (like equipping items, eating food, etc)! - -The throw UI is the one that should know about the inventory and targeting -UIs. Ideally it would set them up, get their "return values" and process those. -How can I send back the values? - -There's actually a really elegant way I came up with for this. At least -I *think* it's elegant. I may end up immuting myself into a corner and -ragequitting this blog series. We'll see. - -Anyway, here's the solution: - -* Make "input" part of the game state. -* Update the game input when you want to return a value from a UI. - -And I can change the game loop to look like this: - -1. Draw the screen. -2. If the game's input is empty, get some from the user to fill it. -3. Process the game to get a new game. The input is now part of the game and - gets processed along with it. -4. Recur with this new game. - -From what little I've used it so far, this method seems very promising. - -Enough design talk. Let's look at the code. - -Implementation --------------- - -In Trystan's tutorial he had three "screens": start, win, and lose. The user -presses keys to transition between them. Not a very fun game, but it lets you -get the game loop up and running before diving into gameplay. - -I did the same thing. Right now everything is in one file because I tend to -code like that until I feel like something needs to be pulled out into its own -namespace. The file is still under a hundred lines of code, so that's not too -bad. - -Let's walk through the code piece by piece. First the namespace: - - :::clojure - (ns caves.core - (:require [lanterna.screen :as s])) - -Next I define some basic data structures: - - :::clojure - (defrecord UI [kind]) - (defrecord World []) - (defrecord Game [world uis input]) - -I used Clojure's records here because I feel like they add a bit of helpful -structure to the code. They're also a bit faster, but that probably won't be -noticeable. You could skip these and just use plain maps if you wanted to, it's -really a personal preference. - -Next is a helper function: - - :::clojure - (defn clear-screen [screen] - (let [blank (apply str (repeat 80 \space))] - (doseq [row (range 24)] - (s/put-string screen 0 row blank)))) - -Unfortunately Lanterna doesn't provide a method for clearing the screen, so -I wrote my own little hacky one that just overwrites everything with spaces. It -assumes the terminal is 80 by 24 characters for now. - -I'll be adding a feature request in the Lanterna issue tracker for this, so -hopefully I'll be able to delete this function in a later post. - -Now to the meaty bits: - - :::clojure - (defmulti draw-ui - (fn [ui game screen] - (:kind ui))) - - (defmethod draw-ui :start [ui game screen] - (s/put-string screen 0 0 "Welcome to the Caves of Clojure!") - (s/put-string screen 0 1 "Press enter to win, anything else to lose.")) - - (defmethod draw-ui :win [ui game screen] - (s/put-string screen 0 0 "Congratulations, you win!") - (s/put-string screen 0 1 "Press escape to exit, anything else to restart.")) - - (defmethod draw-ui :lose [ui game screen] - (s/put-string screen 0 0 "Sorry, better luck next time.") - (s/put-string screen 0 1 "Press escape to exit, anything else to go.")) - - (defn draw-game [game screen] - (clear-screen screen) - (doseq [ui (:uis game)] - (draw-ui ui game screen)) - (s/redraw screen)) - -Here we have the drawing code. - -The UIs are very simple for now. They each just output a couple of lines of -text. None of them actually look at the game state at all, but in the future -some of them will need to do that (e.g.: when showing the list of items in the -player's inventory). - -I made the `draw-ui` function a multimethod to make it easy to define the logic -for each UI separately. Each definition could even live in its own file if -I wanted it to. There are other ways to do this, but I like the concision and -simplicity of this one. - -The `draw-game` function takes the immutable game object and draws some text to -the user's terminal. It's fairly simple. The `redraw` call is needed because -Lanterna [double buffers][] the output. Check out the [clojure-lanterna -documentation][clojure-lanterna] for more information if you're curious. - -[double buffers]: https://en.wikipedia.org/wiki/Multiple_buffering#Double_buffering_in_computer_graphics -[clojure-lanterna]: http://sjl.bitbucket.org/clojure-lanterna/ - - :::clojure - (defmulti process-input - (fn [game input] - (:kind (last (:uis game))))) - - (defmethod process-input :start [game input] - (if (= input :enter) - (assoc game :uis [(new UI :win)]) - (assoc game :uis [(new UI :lose)]))) - - (defmethod process-input :win [game input] - (if (= input :escape) - (assoc game :uis []) - (assoc game :uis [(new UI :start)]))) - - (defmethod process-input :lose [game input] - (if (= input :escape) - (assoc game :uis []) - (assoc game :uis [(new UI :start)]))) - - (defn get-input [game screen] - (assoc game :input (s/get-key-blocking screen))) - -UIs need to know how to process their input. I used a multimethod for this too. - -The method takes the game and the input as parameters and returns a modified -copy of the game object that represents the new state. Currently none of them -use the "returning as input" trick, but we'll see that in one of the next few -posts. - -Notice how the UIs all simply replace the UI stack in the game they return? -This is fine for now, but in the future they'll be more likely to just pop off -the last one (themselves) rather than replace the entire stack. - -An empty UI stack means "quit the game", as we'll see in a moment. - -You'll also see why the input is separate from the game soon. - -The `get-input` function gets a keypress from the user and sticks it into the -game object. Nothing crazy there. - -And now, the game loop: - - :::clojure - (defn run-game [game screen] - (loop [{:keys [input uis] :as game} game] - (when-not (empty? uis) - (draw-game game screen) - (if (nil? input) - (recur (get-input game screen)) - (recur (process-input (dissoc game :input) input)))))) - -Here we go. The `run-game` function `loop`s on a game object each time. - -First: if there are no UIs, we're done and can drop out. Cool. - -If there are UIs, draw the game to the user's terminal. - -Then it checks if it needs to get a keypress from the user. If so, do that, -update the game object, and start again. - -I could make this a bit more efficient by continuing on to process the input -without another round through the loop, but performance probably isn't a concern -at the moment. I'll revisit this in the future if it becomes an issue, but for -now I like this structure. - -Anyway, if we *do* have input (i.e.: either we grabbed a keypress or a UI -returned something the last time through the loop), process it. Remember that -the `process-input` function is a multimethod that dispatches on the `:kind` of -the last UI in the stack. - -Here your can see why `process-input` takes the game and input separately. I -*could* just pass the game and pull out the `:input` value, but then I'd also -need to `dissoc` the input from the modified game object in every UI that didn't -return a value. - -If I didn't `dissoc` the input, the input would always be present and would -cause an infinite loop. You can play around with this by replacing `(dissoc -game :input)` with `game` and watching what happens. - -Next is a simple helper function: - - :::clojure - (defn new-game [] - (new Game - (new World) - [(new UI :start)] - nil)) - -Nothing fancy. You could just inline that body into the next function if you -wanted, but I'm thinking ahead to when I'm going to want to generate a random -world. - -Finally, the bootstrapping: - - :::clojure - (defn main - ([screen-type] (main screen-type false)) - ([screen-type block?] - (letfn [(go [] - (let [screen (s/get-screen screen-type)] - (s/in-screen screen - (run-game (new-game) screen))))] - (if block? - (go) - (future (go)))))) - - (defn -main [& args] - (let [args (set args) - screen-type (cond - (args ":swing") :swing - (args ":text") :text - :else :auto)] - (main screen-type true))) - -`-main` looks almost the same as before, but `main` has changed quite a bit. -What happened? - -The short answer is that most of the change is to work around some Clojure/JVM -silliness. The important bit is that I now create a fresh game object and fire -up the game loop with `(run-game (new-game) screen)`. - -If you're curious about the rest, read [this Clojure bug report][bug]. I wanted -to be able to run the game from the command line as normal, but from a REPL -without blocking the REPL itself, so I could play around with things. - -[bug]: http://dev.clojure.org/jira/browse/CLJ-959 - -That's it! It clocks in at 98 lines of code. Here's the whole file at once: - - :::clojure - (ns caves.core - (:require [lanterna.screen :as s])) - - - ; Data Structures ------------------------------------------------------------- - (defrecord UI [kind]) - (defrecord World []) - (defrecord Game [world uis input]) - - ; Utility Functions ----------------------------------------------------------- - (defn clear-screen [screen] - (let [blank (apply str (repeat 80 \space))] - (doseq [row (range 24)] - (s/put-string screen 0 row blank)))) - - - ; Drawing --------------------------------------------------------------------- - (defmulti draw-ui - (fn [ui game screen] - (:kind ui))) - - (defmethod draw-ui :start [ui game screen] - (s/put-string screen 0 0 "Welcome to the Caves of Clojure!") - (s/put-string screen 0 1 "Press enter to win, anything else to lose.")) - - (defmethod draw-ui :win [ui game screen] - (s/put-string screen 0 0 "Congratulations, you win!") - (s/put-string screen 0 1 "Press escape to exit, anything else to restart.")) - - (defmethod draw-ui :lose [ui game screen] - (s/put-string screen 0 0 "Sorry, better luck next time.") - (s/put-string screen 0 1 "Press escape to exit, anything else to go.")) - - (defn draw-game [game screen] - (clear-screen screen) - (doseq [ui (:uis game)] - (draw-ui ui game screen)) - (s/redraw screen)) - - - ; Input ----------------------------------------------------------------------- - (defmulti process-input - (fn [game input] - (:kind (last (:uis game))))) - - (defmethod process-input :start [game input] - (if (= input :enter) - (assoc game :uis [(new UI :win)]) - (assoc game :uis [(new UI :lose)]))) - - (defmethod process-input :win [game input] - (if (= input :escape) - (assoc game :uis []) - (assoc game :uis [(new UI :start)]))) - - (defmethod process-input :lose [game input] - (if (= input :escape) - (assoc game :uis []) - (assoc game :uis [(new UI :start)]))) - - (defn get-input [game screen] - (assoc game :input (s/get-key-blocking screen))) - - - ; Main ------------------------------------------------------------------------ - (defn run-game [game screen] - (loop [{:keys [input uis] :as game} game] - (when-not (empty? uis) - (draw-game game screen) - (if (nil? input) - (recur (get-input game screen)) - (recur (process-input (dissoc game :input) input)))))) - - (defn new-game [] - (new Game - (new World) - [(new UI :start)] - nil)) - - (defn main - ([screen-type] (main screen-type false)) - ([screen-type block?] - (letfn [(go [] - (let [screen (s/get-screen screen-type)] - (s/in-screen screen - (run-game (new-game) screen))))] - (if block? - (go) - (future (go)))))) - - - (defn -main [& args] - (let [args (set args) - screen-type (cond - (args ":swing") :swing - (args ":text") :text - :else :auto)] - (main screen-type true))) - -And here are some screenshots: - -![Screenshot](/media/images{{ parent_url }}/caves-02-01.png) - -![Screenshot](/media/images{{ parent_url }}/caves-02-02.png) - -![Screenshot](/media/images{{ parent_url }}/caves-02-03.png) - -It's not a very exciting game yet, but it all works, and I've managed to use an -immutable data structure of basic maps and records to represent everything -I need. - -The drawing functions aren't "pure" in the "no I/O" sense, but they're kind of -pure in another way -- they take an immutable data structure and draw something -to the screen based solely on that. I think this is going to make things easy -to work with down the line. - -Testing -------- - -I'll leave you with one final tidbit to read through if you want more. - -Encapsulating the game state as an immutable objects means I can test actions -and their effects on the world individually, without a game loop: - - :::clojure - (ns caves.core-test - (:import [caves.core UI World Game]) - (:use clojure.test - caves.core)) - - (defn current-ui [game] - (:kind (last (:uis game)))) - - - (deftest test-start - (let [game (new Game nil [(new UI :start)] nil)] - - (testing "Enter wins at the starting screen." - (let [result (process-input game :enter)] - (is (= (current-ui result) :win)))) - - (testing "Other keys lose at the starting screen." - (let [results (map (partial process-input game) - [\space \a \A :escape :up :backspace])] - (doseq [result results] - (is (= (current-ui result) :lose))))))) - -That's pretty cool! - -{% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/07/caves-of-clojure-02.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2012/07/caves-of-clojure-02.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,593 @@ ++++ +title = "The Caves of Clojure: Part 2" +snip = "Dealing with state." +date = 2012-07-08T9:26:00Z +draft = false + ++++ + +This post is part of an ongoing series. If you haven't already done so, you +should probably start at [the beginning][]. + +This entry corresponds to [post two in Trystan's tutorial][trystan-tut]. + +If you want to follow along, the code for the series is [on Bitbucket][bb] and +[on GitHub][gh]. Update to the `entry-02` tag to see the code as it stands +after this post. + +[the beginning]: /blog/2012/07/caves-of-clojure-01/ +[trystan-tut]: http://trystans.blogspot.com/2011/08/roguelike-tutorial-02-input-output.html +[bb]: http://bitbucket.org/sjl/caves/ +[gh]: http://github.com/sjl/caves/ + +{{% toc %}} + +Summary +------- + +In Trystan's second post he introduces the concept of the game loop, as well as +what he calls "screens": objects that handle drawing the interface and +processing user input. + +I could try to port his design directly over to Clojure, but instead I wanted to +step back and see if I could find a way to make things more functional. + +I think I've figured out a way to make it work, so I'm going to implement that. + +State +----- + +When I first started thinking about how to model the game's state and the main +game loop I had lots of crazy ideas bouncing around in my head. Most of them +involved an immutable world (immutable! good!) and agents representing Trystan's +"screens". + +The more I thought about it, though, the more it looked like the agents would +wind up being a tangled mess. I put down the keyboard, took a shower, had +dinner with a friend, and let the problem roll around in my head for a bit. + +At some point the following train of thought happened somewhere in my brain: + +* The immutable "state" that I keep should contain *everything* needed to render + the game on the user's screen. +* I originally thought I'd need to track the "world" as the state, but the world + isn't enough! +* In addition to the world, the user interface (menus, stats, etc) is also + rendered. + +So instead of keeping a "world" as my state, I'm going to keep a "game". + +The User Interface +------------------ + +If I'm going to keep track of the user interface in the "game" state, I need +a way to represent it. + +There are two halves to the user interface: "input" and "output". First let's +consider output. + +Trystan's screens are objects that handle their own drawing. At any given time +there's one "active" screen object, which gets asked to draw itself. If you +peek ahead in his tutorial you'll see that he ends up introducing a "subscreen" +concept to get screens layered on top of each other. + +Instead of having a single active screen with subscreens, I decided to keep +a flat vector of screens. The last screen in the vector is the "active" one, +and is effectively a subscreen of the one that comes before it. + +At this point I'm going to switch terms. Unfortunately Lanterna uses the word +"screen" to mean something and I didn't want to try to keep two separate +concepts under the same word, so in the code I called screens "UIs", and from +now on I'll be using that word. + +So what *is* a UI in the code? Well, it's basically just a map with a `:kind` +mapping specifying what kind of UI it is! It also might have some extra keys +and values to represent its state too. + +For example, at some point I expect to have a UI stack that looks something like +this: + +```clojure +[{:kind :play} + {:kind :throw} + {:kind :inventory-select}] +``` + +This would be the UI stack when you were throwing something in your inventory at +a monster and were choosing what to throw. Once you pick an item you'd need to +target a monster, so the stack would become: + +```clojure +[{:kind :play} + {:kind :throw :item foo} + {:kind :target}] +``` + +Now the last (i.e.: "active") UI is the targetting UI, but also notice that the +throwing UI has a bit of state attached to it now (the item the user picked). +We'll talk about how that got there a bit later. + +As I said before, the "state" for our game is going to be a "game", which +consists of the world and the user interface. So our "game" object is going to +be a map that looks something like this: + +```clojure +{:world {} + :uis [,,,]} +``` + +For now the `:world` is empty. In the future it'll contain stuff like the tiles +of the map, the monsters, the player, and lots of other stuff. The `:uis` is +the UI stack. + +Between the two I have enough information to draw the game fully to the user's +terminal by doing something like: `(map #(draw-ui ui game) (:uis game))`. We'll +see the real code shortly, but that's actually pretty close. + +User Input +---------- + +In an imperative programming style our game loop would look something like this: + +1. Draw the screen. +2. Get some input from the user. +3. Process that input, modifying the world. +4. GOTO 1. + +In this functional loop, I want it to look more like this: + +1. Draw the screen. +2. Get some user input. +3. Process the input and the game to get a new game. +4. Recur with this new game. + +How do I handle user input? Well it depends on the current UI — pressing `d` +at the main screen will do something different than pressing it in an inventory +selection screen, for example. + +So the UIs need to know how to handle input. There are a number of different +ways I can do that. One option might be to have `:handle-input (fn ...)` as +part of the UI. I chose a different route which you'll see below, but that's +not important for now. + +The important part is that one I glossed over in the last section. How do I go +from this: + +```clojure +[{:kind :play} + {:kind :throw} + {:kind :inventory-select}] +``` + +to this: + +```clojure +[{:kind :play} + {:kind :throw :item foo} + {:kind :target}] +``` + +Let's follow the proposed game loop and see what happens. + +1. Draw the play UI, then the throw UI, then the inventory UI. +2. Get a keypress from the user. +3. Give that keypress and the game itself to the UI input handling function to + get a new game. +4. Recur with this new game. + +Step three is the tricky part. What does the inventory handler need to do to +give back a new game? + +It would need to pop itself off the UI stack (which is okay), put the selected +item in the previous UI (a bit scary, but probably not a problem in practice), +and create the targeting UI. + +This last part is a deal breaker. The inventory selection UI shouldn't know +anything about the targeting UI, because then I won't be able to reuse it for +other functions (like equipping items, eating food, etc)! + +The throw UI is the one that should know about the inventory and targeting +UIs. Ideally it would set them up, get their "return values" and process those. +How can I send back the values? + +There's actually a really elegant way I came up with for this. At least +I *think* it's elegant. I may end up immuting myself into a corner and +ragequitting this blog series. We'll see. + +Anyway, here's the solution: + +* Make "input" part of the game state. +* Update the game input when you want to return a value from a UI. + +And I can change the game loop to look like this: + +1. Draw the screen. +2. If the game's input is empty, get some from the user to fill it. +3. Process the game to get a new game. The input is now part of the game and + gets processed along with it. +4. Recur with this new game. + +From what little I've used it so far, this method seems very promising. + +Enough design talk. Let's look at the code. + +Implementation +-------------- + +In Trystan's tutorial he had three "screens": start, win, and lose. The user +presses keys to transition between them. Not a very fun game, but it lets you +get the game loop up and running before diving into gameplay. + +I did the same thing. Right now everything is in one file because I tend to +code like that until I feel like something needs to be pulled out into its own +namespace. The file is still under a hundred lines of code, so that's not too +bad. + +Let's walk through the code piece by piece. First the namespace: + +```clojure +(ns caves.core + (:require [lanterna.screen :as s])) +``` + +Next I define some basic data structures: + +```clojure +(defrecord UI [kind]) +(defrecord World []) +(defrecord Game [world uis input]) +``` + +I used Clojure's records here because I feel like they add a bit of helpful +structure to the code. They're also a bit faster, but that probably won't be +noticeable. You could skip these and just use plain maps if you wanted to, it's +really a personal preference. + +Next is a helper function: + +```clojure +(defn clear-screen [screen] + (let [blank (apply str (repeat 80 \space))] + (doseq [row (range 24)] + (s/put-string screen 0 row blank)))) +``` + +Unfortunately Lanterna doesn't provide a method for clearing the screen, so +I wrote my own little hacky one that just overwrites everything with spaces. It +assumes the terminal is 80 by 24 characters for now. + +I'll be adding a feature request in the Lanterna issue tracker for this, so +hopefully I'll be able to delete this function in a later post. + +Now to the meaty bits: + +```clojure +(defmulti draw-ui + (fn [ui game screen] + (:kind ui))) + +(defmethod draw-ui :start [ui game screen] + (s/put-string screen 0 0 "Welcome to the Caves of Clojure!") + (s/put-string screen 0 1 "Press enter to win, anything else to lose.")) + +(defmethod draw-ui :win [ui game screen] + (s/put-string screen 0 0 "Congratulations, you win!") + (s/put-string screen 0 1 "Press escape to exit, anything else to restart.")) + +(defmethod draw-ui :lose [ui game screen] + (s/put-string screen 0 0 "Sorry, better luck next time.") + (s/put-string screen 0 1 "Press escape to exit, anything else to go.")) + +(defn draw-game [game screen] + (clear-screen screen) + (doseq [ui (:uis game)] + (draw-ui ui game screen)) + (s/redraw screen)) +``` + +Here we have the drawing code. + +The UIs are very simple for now. They each just output a couple of lines of +text. None of them actually look at the game state at all, but in the future +some of them will need to do that (e.g.: when showing the list of items in the +player's inventory). + +I made the `draw-ui` function a multimethod to make it easy to define the logic +for each UI separately. Each definition could even live in its own file if +I wanted it to. There are other ways to do this, but I like the concision and +simplicity of this one. + +The `draw-game` function takes the immutable game object and draws some text to +the user's terminal. It's fairly simple. The `redraw` call is needed because +Lanterna [double buffers][] the output. Check out the [clojure-lanterna +documentation][clojure-lanterna] for more information if you're curious. + +[double buffers]: https://en.wikipedia.org/wiki/Multiple_buffering#Double_buffering_in_computer_graphics +[clojure-lanterna]: http://sjl.bitbucket.org/clojure-lanterna/ + +```clojure +(defmulti process-input + (fn [game input] + (:kind (last (:uis game))))) + +(defmethod process-input :start [game input] + (if (= input :enter) + (assoc game :uis [(new UI :win)]) + (assoc game :uis [(new UI :lose)]))) + +(defmethod process-input :win [game input] + (if (= input :escape) + (assoc game :uis []) + (assoc game :uis [(new UI :start)]))) + +(defmethod process-input :lose [game input] + (if (= input :escape) + (assoc game :uis []) + (assoc game :uis [(new UI :start)]))) + +(defn get-input [game screen] + (assoc game :input (s/get-key-blocking screen))) +``` + +UIs need to know how to process their input. I used a multimethod for this too. + +The method takes the game and the input as parameters and returns a modified +copy of the game object that represents the new state. Currently none of them +use the "returning as input" trick, but we'll see that in one of the next few +posts. + +Notice how the UIs all simply replace the UI stack in the game they return? +This is fine for now, but in the future they'll be more likely to just pop off +the last one (themselves) rather than replace the entire stack. + +An empty UI stack means "quit the game", as we'll see in a moment. + +You'll also see why the input is separate from the game soon. + +The `get-input` function gets a keypress from the user and sticks it into the +game object. Nothing crazy there. + +And now, the game loop: + +```clojure +(defn run-game [game screen] + (loop [{:keys [input uis] :as game} game] + (when-not (empty? uis) + (draw-game game screen) + (if (nil? input) + (recur (get-input game screen)) + (recur (process-input (dissoc game :input) input)))))) +``` + +Here we go. The `run-game` function `loop`s on a game object each time. + +First: if there are no UIs, we're done and can drop out. Cool. + +If there are UIs, draw the game to the user's terminal. + +Then it checks if it needs to get a keypress from the user. If so, do that, +update the game object, and start again. + +I could make this a bit more efficient by continuing on to process the input +without another round through the loop, but performance probably isn't a concern +at the moment. I'll revisit this in the future if it becomes an issue, but for +now I like this structure. + +Anyway, if we *do* have input (i.e.: either we grabbed a keypress or a UI +returned something the last time through the loop), process it. Remember that +the `process-input` function is a multimethod that dispatches on the `:kind` of +the last UI in the stack. + +Here your can see why `process-input` takes the game and input separately. I +*could* just pass the game and pull out the `:input` value, but then I'd also +need to `dissoc` the input from the modified game object in every UI that didn't +return a value. + +If I didn't `dissoc` the input, the input would always be present and would +cause an infinite loop. You can play around with this by replacing `(dissoc +game :input)` with `game` and watching what happens. + +Next is a simple helper function: + +```clojure +(defn new-game [] + (new Game + (new World) + [(new UI :start)] + nil)) +``` + +Nothing fancy. You could just inline that body into the next function if you +wanted, but I'm thinking ahead to when I'm going to want to generate a random +world. + +Finally, the bootstrapping: + +```clojure +(defn main + ([screen-type] (main screen-type false)) + ([screen-type block?] + (letfn [(go [] + (let [screen (s/get-screen screen-type)] + (s/in-screen screen + (run-game (new-game) screen))))] + (if block? + (go) + (future (go)))))) + +(defn -main [& args] + (let [args (set args) + screen-type (cond + (args ":swing") :swing + (args ":text") :text + :else :auto)] + (main screen-type true))) +``` + +`-main` looks almost the same as before, but `main` has changed quite a bit. +What happened? + +The short answer is that most of the change is to work around some Clojure/JVM +silliness. The important bit is that I now create a fresh game object and fire +up the game loop with `(run-game (new-game) screen)`. + +If you're curious about the rest, read [this Clojure bug report][bug]. I wanted +to be able to run the game from the command line as normal, but from a REPL +without blocking the REPL itself, so I could play around with things. + +[bug]: http://dev.clojure.org/jira/browse/CLJ-959 + +That's it! It clocks in at 98 lines of code. Here's the whole file at once: + +```clojure +(ns caves.core + (:require [lanterna.screen :as s])) + + +; Data Structures ------------------------------------------------------------- +(defrecord UI [kind]) +(defrecord World []) +(defrecord Game [world uis input]) + +; Utility Functions ----------------------------------------------------------- +(defn clear-screen [screen] + (let [blank (apply str (repeat 80 \space))] + (doseq [row (range 24)] + (s/put-string screen 0 row blank)))) + + +; Drawing --------------------------------------------------------------------- +(defmulti draw-ui + (fn [ui game screen] + (:kind ui))) + +(defmethod draw-ui :start [ui game screen] + (s/put-string screen 0 0 "Welcome to the Caves of Clojure!") + (s/put-string screen 0 1 "Press enter to win, anything else to lose.")) + +(defmethod draw-ui :win [ui game screen] + (s/put-string screen 0 0 "Congratulations, you win!") + (s/put-string screen 0 1 "Press escape to exit, anything else to restart.")) + +(defmethod draw-ui :lose [ui game screen] + (s/put-string screen 0 0 "Sorry, better luck next time.") + (s/put-string screen 0 1 "Press escape to exit, anything else to go.")) + +(defn draw-game [game screen] + (clear-screen screen) + (doseq [ui (:uis game)] + (draw-ui ui game screen)) + (s/redraw screen)) + + +; Input ----------------------------------------------------------------------- +(defmulti process-input + (fn [game input] + (:kind (last (:uis game))))) + +(defmethod process-input :start [game input] + (if (= input :enter) + (assoc game :uis [(new UI :win)]) + (assoc game :uis [(new UI :lose)]))) + +(defmethod process-input :win [game input] + (if (= input :escape) + (assoc game :uis []) + (assoc game :uis [(new UI :start)]))) + +(defmethod process-input :lose [game input] + (if (= input :escape) + (assoc game :uis []) + (assoc game :uis [(new UI :start)]))) + +(defn get-input [game screen] + (assoc game :input (s/get-key-blocking screen))) + + +; Main ------------------------------------------------------------------------ +(defn run-game [game screen] + (loop [{:keys [input uis] :as game} game] + (when-not (empty? uis) + (draw-game game screen) + (if (nil? input) + (recur (get-input game screen)) + (recur (process-input (dissoc game :input) input)))))) + +(defn new-game [] + (new Game + (new World) + [(new UI :start)] + nil)) + +(defn main + ([screen-type] (main screen-type false)) + ([screen-type block?] + (letfn [(go [] + (let [screen (s/get-screen screen-type)] + (s/in-screen screen + (run-game (new-game) screen))))] + (if block? + (go) + (future (go)))))) + + +(defn -main [& args] + (let [args (set args) + screen-type (cond + (args ":swing") :swing + (args ":text") :text + :else :auto)] + (main screen-type true))) +``` + +And here are some screenshots: + +![Screenshot](/media/images/blog/2012/07/caves-02-01.png) + +![Screenshot](/media/images/blog/2012/07/caves-02-02.png) + +![Screenshot](/media/images/blog/2012/07/caves-02-03.png) + +It's not a very exciting game yet, but it all works, and I've managed to use an +immutable data structure of basic maps and records to represent everything +I need. + +The drawing functions aren't "pure" in the "no I/O" sense, but they're kind of +pure in another way — they take an immutable data structure and draw something +to the screen based solely on that. I think this is going to make things easy +to work with down the line. + +Testing +------- + +I'll leave you with one final tidbit to read through if you want more. + +Encapsulating the game state as an immutable objects means I can test actions +and their effects on the world individually, without a game loop: + +```clojure +(ns caves.core-test + (:import [caves.core UI World Game]) + (:use clojure.test + caves.core)) + +(defn current-ui [game] + (:kind (last (:uis game)))) + + +(deftest test-start + (let [game (new Game nil [(new UI :start)] nil)] + + (testing "Enter wins at the starting screen." + (let [result (process-input game :enter)] + (is (= (current-ui result) :win)))) + + (testing "Other keys lose at the starting screen." + (let [results (map (partial process-input game) + [\space \a \A :escape :up :backspace])] + (doseq [result results] + (is (= (current-ui result) :lose))))))) +``` + +That's pretty cool! + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/07/caves-of-clojure-03-1.html --- a/content/blog/2012/07/caves-of-clojure-03-1.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,363 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "The Caves of Clojure: Part 3.1" - snip: "World generation." - created: 2012-07-09 9:37:00 - %} - -{% block article %} - -This post is part of an ongoing series. If you haven't already done so, you -should probably start at [the beginning][]. - -This entry corresponds to [post three in Trystan's tutorial][trystan-tut]. - -If you want to follow along, the code for the series is [on Bitbucket][bb] and -[on GitHub][gh]. Update to the `entry-03-1` tag to see the code as it stands -after this post. - -[the beginning]: /blog/2012/07/caves-of-clojure-01/ -[trystan-tut]: http://trystans.blogspot.com/2011/08/roguelike-tutorial-03-scrolling-through.html -[bb]: http://bitbucket.org/sjl/caves/ -[gh]: http://github.com/sjl/caves/ - -[TOC] - -Summary -------- - -In Trystan's third post he introduces three new things: - -* Random world generation. -* Smoothing the world to look nicer. -* The "play screen" and scrolling. - -I'm going to split this up into three separate short posts, so you can see the -process I actually went through as I built it. This post will deal with the -world generation and basic displaying. The next one will be about smoothing, -and the one after that about scrolling around. - -Organization ------------- - -First of all, my single `core.clj` is going to get a bit crowded by the time -we're done with this, so it's time to start splitting it apart. I made -a `world.clj` file alongside `core.clj`. That'll do for now. - -Creating a Random World ------------------------ - -First I set up the namespace: - - :::clojure - (ns caves.world) - -Next I defined a constant for the size of the world. I chose 160 by 50 tiles -arbitrarily: - - :::clojure - (def world-size [160 50]) - -I moved the `World` record declaration out of `core.clj` and into this file: - - :::clojure - (defrecord World [tiles]) - -I also added the `tiles` attribute to it. For now I'm going to store the tiles -as a vector of rows, where each row is a vector of tiles. - -What is a tile? For now it's going to be a simple record: - - :::clojure - (defrecord Tile [kind glyph color]) - -Tiles are immutable, so let's make a map of some of the ones we'll be needing: - - :::clojure - (def tiles - {:floor (new Tile :floor "." :white) - :wall (new Tile :wall "#" :white) - :bound (new Tile :bound "X" :black)}) - -The `:bound` Tile represents "out of bounds". It will be returned if you try to -get a tile outside the bounds of the map. There are other ways to handle that, -but this is the one Trystan used so I'm going to use it too. - -Next I created a little helper function for retrieving a specific tile: - - :::clojure - (defn get-tile [tiles x y] - (get-in tiles [y x] (:bound tiles))) - -Remember that we're storing the tiles as a vector of rows, so when we index into -it we need to get the y coordinate (i.e.: the row) first, then index in for the -column with the x coordinate. - -This is a bit ugly, but storing the screen as rows is going to help us later as -we draw the screen, and it gives us the opportunity to do bounds checking as -well. - -The `get-in` call is a really handy way to check bounds. No worrying about -comparing indexes to dimensions -- just try to get it and if you fail it must be -out of bounds. - -That's about it for the world structure. Time to move to the generation: - - :::clojure - (defn random-tiles [] - (let [[cols rows] world-size] - (letfn [(random-tile [] - (tiles (rand-nth [:floor :wall]))) - (random-row [] - (vec (repeatedly cols random-tile)))] - (vec (repeatedly rows random-row))))) - - (defn random-world [] - (new World (random-tiles))) - -Nothing too fancy happening here. In `random-tiles` I built up a series of -helper functions to make it easier to read. You could save some LOC by just -using anonymous functions instead, but to me this way is easier to read. -Personal preference, I guess. - -For now we're just going to generate a world where every tile has an equal -chance of being a wall or a floor. I might revisit this later if I want to make -the caves sparser or denser. - -That's it for the world generation. Next we'll move on to actually displaying -the new random worlds. - -Displaying ----------- - -Let's switch back to `core.clj`. First I updated the namespace to pull in the -`random-world` function: - - :::clojure - (ns caves.core - (:use [caves.world :only [random-world]]) - (:require [lanterna.screen :as s])) - -Before going further I decided to do a bit of cleanup. Instead of hardcoding -the 80 by 24 terminal size, I pulled it out into a constant: - - :::clojure - (def screen-size [80 24]) - -I updated `clear-screen` to use that: - - :::clojure - (defn clear-screen [screen] - (let [[cols rows] screen-size - blank (apply str (repeat cols \space))] - (doseq [row (range rows)] - (s/put-string screen 0 row blank)))) - -It's still not perfect (what if the user's terminal isn't 80 by 24?) but it's -not something I care enough to fix right now. I'll get to it later. At least -now the hardcoded numbers are in one spot. - -There's a few things I need to do to get the world on the screen. First -I created a `:play` UI similar to Trystan's. I'm not a big fan of the -generic-sounding name, but I couldn't come up with anything better in a few -minutes of thinking. - -Creating a UI requires implementing the `draw-ui` and `process-input` -multimethods from the previous post. I'll start with the easy one: input -processing. - -For now the flow of the game will go like this: - -1. The player is shown an introduction screen with some instructions. -2. They press a key and see the world. -3. Pressing enter wins the game. Backspace loses the game. Any other key does - nothing. -4. Once they win or lose, they see a text blurb and can press escape to quit, or - any other key to GOTO 1. - -With that in mind, I wrote the `process-input` implementation for `:play` UIs: - - :::clojure - (defmethod process-input :play [game input] - (case input - :enter (assoc game :uis [(new UI :win)]) - :backspace (assoc game :uis [(new UI :lose)]) - game)) - -I'm still replacing the entire UI stack at once. I'm going to be throwing that -code away later anyway so it's not a big deal. - -Now I need to update the `:start` UI to send the user to the `:play` UI instead -of directly to the `:win` or `:lose` UIs. - - :::clojure - (defmethod process-input :start [game input] - (-> game - (assoc :world (random-world)) - (assoc :uis [(new UI :play)]))) - -I also decided that this is where I'd generate the new random world. It makes -sense to put that here, because every time you restart the game you should get -a different world. - -On the other hand, this means that the `process-input` function is no longer -pure for `:start` UIs (all the other input processing functions are still pure). -I'm not sure how I feel about that. For now I'm going to accept it, but I may -rethink it in the future. - -Now that I have the world generation in there, I can remove the `(new World)` -call from the `new-game` helper function: - - :::clojure - (defn new-game [] - (new Game nil [(new UI :start)] nil)) - -Now `new-game` does even less than before. It just sets up a game object with -the initial UI and `nil` world/input. - -Okay, on to the last piece: drawing the world. This is some pretty dense code, -but don't be scared -- I'll guide you through it and we'll make it together: - - :::clojure - (defmethod draw-ui :play [ui {{:keys [tiles]} :world :as game} screen] - (let [[cols rows] screen-size - vcols cols - vrows (dec rows) - start-x 0 - start-y 0 - end-x (+ start-x vcols) - end-y (+ start-y vrows)] - (doseq [[vrow-idx mrow-idx] (map vector - (range 0 vrows) - (range start-y end-y)) - :let [row-tiles (subvec (tiles mrow-idx) start-x end-x)]] - (doseq [vcol-idx (range vcols) - :let [{:keys [glyph color]} (row-tiles vcol-idx)]] - (s/put-string screen vcol-idx vrow-idx glyph {:fg color}))))) - -Let's look at this beast line-by-line. First we pull the tile vector out of the -game object with Clojure's destructuring: - - :::clojure - {{:keys [tiles]} :world :as game} - -This seems a bit hard to read to me, so I might move it into the `let` statement -with a `get-in` call later. Maybe. - -Next I bind a bunch of symbols I'll be using: - - :::clojure - (let [[cols rows] screen-size - vcols cols - vrows (dec rows) - start-x 0 - start-y 0 - end-x (+ start-x vcols) - end-y (+ start-y vrows)] - ,,,) - -`cols` and `rows` are the dimensions of the screen (hardcoded at 80 by 24 for -the moment). - -`vcols` and `vrows` are the "viewport columns" and "viewport rows". The -"viewport" is what I'm calling the part of the screen that's actually showing -the world. I'm reserving one row at the bottom of the screen to use for -displaying the player's hit points, score, and so on. It would be trivial to -increase that to two rows if I need more later. - -`start-x` and `start-y` are the coordinates of the upper-left corner of the -viewport in the full map, and `end-x` and `end-y` are the coordinates of the -lower-right corner. For now I'm just displaying the upper-left section of the -map. In the entry after the next one I'll add the ability to scroll around. - -It's easier to explain with a diagram. Imagine I reduced the size of the world -to 10 by 10 and the terminal to 5 by 3, and the user was standing near the -middle of the map: - - :::text - columns (x) - 0123456789 - rows 0.......... - (y) 1.......... - 2.......... - 3..VVVVV... - 4..VVVVV... - 5..VVVVV... - 6.......... - 7.......... - 8.......... - 9.......... - -Here `V` represents the portion of the map the viewport can show (which is what -we'll be drawing to the user's terminal). In this example `start-x` would -be `2`, `start-y` would be `3`, `end-x` would be `6`, and `end-y` would be `5`. - -Okay, so I've calculated the part of the map that needs to be drawn. Now -I loop through the rows: - - :::clojure - (doseq [[vrow-idx mrow-idx] (map vector - (range 0 vrows) - (range start-y end-y)) - :let [row-tiles (subvec (tiles mrow-idx) start-x end-x)]] - ,,,) - -This is a bit obtuse, but basically the `map` call pairs up the viewport row and -map row indices. In our example it would result in this: - - :::clojure - [[0 3] - [1 4] - [2 5]] - -So viewport row `1` corresponds to map row `4`. There's probably a less -"clever" way to do this that I should use instead. - -For each row, I grab the tiles we're going to draw and store them in a vector -called `row-tiles` by grabbing the row of tiles from the world and taking -a slice of it. - -Almost there! Next I loop through each column in the row: - - :::clojure - (doseq [vcol-idx (range vcols) - :let [{:keys [glyph color]} (row-tiles vcol-idx)]] - ,,,) - -For each column I grab the appropriate tile and figure out what glyph and color -to draw (remember the definition of `Tile`: `(defrecord Tile [kind glyph -color])`). - -Finally I can actually draw the tile to the screen at the appropriate place: - - :::clojure - (s/put-string screen vcol-idx vrow-idx glyph {:fg color}) - -Whew! If that seemed painful and fiddly to you, trust me, I agree. I'm open to -suggestions on making it easier to read. - -Results -------- - -Now that the `:play` UI knows how to draw itself and process its input, and is -properly hooked up by the `:start` UI, it's time to give it a shot! - -![Screenshot](/media/images{{ parent_url }}/caves-03-1-01.png) - -![Screenshot](/media/images{{ parent_url }}/caves-03-1-02.png) - -![Screenshot](/media/images{{ parent_url }}/caves-03-1-03.png) - -Each time we start we get a different random world. Great! - -The code is getting a bit big to include in its entirety, but you can view it -[on GitHub][result-code]. - -[result-code]: https://github.com/sjl/caves/tree/entry-03-1/src/caves - -That covers the first part of Trystan's third post. Aside from the painful -`draw-ui` function it was pretty easy to add. In the next post I'll add the -smoothing code to make the caves look a bit nicer. - -{% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/07/caves-of-clojure-03-1.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2012/07/caves-of-clojure-03-1.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,381 @@ ++++ +title = "The Caves of Clojure: Part 3.1" +snip = "World generation." +date = 2012-07-09T9:37:00Z +draft = false + ++++ + +This post is part of an ongoing series. If you haven't already done so, you +should probably start at [the beginning][]. + +This entry corresponds to [post three in Trystan's tutorial][trystan-tut]. + +If you want to follow along, the code for the series is [on Bitbucket][bb] and +[on GitHub][gh]. Update to the `entry-03-1` tag to see the code as it stands +after this post. + +[the beginning]: /blog/2012/07/caves-of-clojure-01/ +[trystan-tut]: http://trystans.blogspot.com/2011/08/roguelike-tutorial-03-scrolling-through.html +[bb]: http://bitbucket.org/sjl/caves/ +[gh]: http://github.com/sjl/caves/ + +{{% toc %}} + +Summary +------- + +In Trystan's third post he introduces three new things: + +* Random world generation. +* Smoothing the world to look nicer. +* The "play screen" and scrolling. + +I'm going to split this up into three separate short posts, so you can see the +process I actually went through as I built it. This post will deal with the +world generation and basic displaying. The next one will be about smoothing, +and the one after that about scrolling around. + +Organization +------------ + +First of all, my single `core.clj` is going to get a bit crowded by the time +we're done with this, so it's time to start splitting it apart. I made +a `world.clj` file alongside `core.clj`. That'll do for now. + +Creating a Random World +----------------------- + +First I set up the namespace: + +```clojure +(ns caves.world) +``` + +Next I defined a constant for the size of the world. I chose 160 by 50 tiles +arbitrarily: + +```clojure +(def world-size [160 50]) +``` + +I moved the `World` record declaration out of `core.clj` and into this file: + +```clojure +(defrecord World [tiles]) +``` + +I also added the `tiles` attribute to it. For now I'm going to store the tiles +as a vector of rows, where each row is a vector of tiles. + +What is a tile? For now it's going to be a simple record: + +```clojure +(defrecord Tile [kind glyph color]) +``` + +Tiles are immutable, so let's make a map of some of the ones we'll be needing: + +```clojure +(def tiles + {:floor (new Tile :floor "." :white) + :wall (new Tile :wall "#" :white) + :bound (new Tile :bound "X" :black)}) +``` + +The `:bound` Tile represents "out of bounds". It will be returned if you try to +get a tile outside the bounds of the map. There are other ways to handle that, +but this is the one Trystan used so I'm going to use it too. + +Next I created a little helper function for retrieving a specific tile: + +```clojure +(defn get-tile [tiles x y] + (get-in tiles [y x] (:bound tiles))) +``` + +Remember that we're storing the tiles as a vector of rows, so when we index into +it we need to get the y coordinate (i.e.: the row) first, then index in for the +column with the x coordinate. + +This is a bit ugly, but storing the screen as rows is going to help us later as +we draw the screen, and it gives us the opportunity to do bounds checking as +well. + +The `get-in` call is a really handy way to check bounds. No worrying about +comparing indexes to dimensions — just try to get it and if you fail it must be +out of bounds. + +That's about it for the world structure. Time to move to the generation: + +```clojure +(defn random-tiles [] + (let [[cols rows] world-size] + (letfn [(random-tile [] + (tiles (rand-nth [:floor :wall]))) + (random-row [] + (vec (repeatedly cols random-tile)))] + (vec (repeatedly rows random-row))))) + +(defn random-world [] + (new World (random-tiles))) +``` + +Nothing too fancy happening here. In `random-tiles` I built up a series of +helper functions to make it easier to read. You could save some LOC by just +using anonymous functions instead, but to me this way is easier to read. +Personal preference, I guess. + +For now we're just going to generate a world where every tile has an equal +chance of being a wall or a floor. I might revisit this later if I want to make +the caves sparser or denser. + +That's it for the world generation. Next we'll move on to actually displaying +the new random worlds. + +Displaying +---------- + +Let's switch back to `core.clj`. First I updated the namespace to pull in the +`random-world` function: + +```clojure +(ns caves.core + (:use [caves.world :only [random-world]]) + (:require [lanterna.screen :as s])) +``` + +Before going further I decided to do a bit of cleanup. Instead of hardcoding +the 80 by 24 terminal size, I pulled it out into a constant: + +```clojure +(def screen-size [80 24]) +``` + +I updated `clear-screen` to use that: + +```clojure +(defn clear-screen [screen] + (let [[cols rows] screen-size + blank (apply str (repeat cols \space))] + (doseq [row (range rows)] + (s/put-string screen 0 row blank)))) +``` + +It's still not perfect (what if the user's terminal isn't 80 by 24?) but it's +not something I care enough to fix right now. I'll get to it later. At least +now the hardcoded numbers are in one spot. + +There's a few things I need to do to get the world on the screen. First +I created a `:play` UI similar to Trystan's. I'm not a big fan of the +generic-sounding name, but I couldn't come up with anything better in a few +minutes of thinking. + +Creating a UI requires implementing the `draw-ui` and `process-input` +multimethods from the previous post. I'll start with the easy one: input +processing. + +For now the flow of the game will go like this: + +1. The player is shown an introduction screen with some instructions. +2. They press a key and see the world. +3. Pressing enter wins the game. Backspace loses the game. Any other key does + nothing. +4. Once they win or lose, they see a text blurb and can press escape to quit, or + any other key to GOTO 1. + +With that in mind, I wrote the `process-input` implementation for `:play` UIs: + +```clojure +(defmethod process-input :play [game input] + (case input + :enter (assoc game :uis [(new UI :win)]) + :backspace (assoc game :uis [(new UI :lose)]) + game)) +``` + +I'm still replacing the entire UI stack at once. I'm going to be throwing that +code away later anyway so it's not a big deal. + +Now I need to update the `:start` UI to send the user to the `:play` UI instead +of directly to the `:win` or `:lose` UIs. + +```clojure +(defmethod process-input :start [game input] + (-> game + (assoc :world (random-world)) + (assoc :uis [(new UI :play)]))) +``` + +I also decided that this is where I'd generate the new random world. It makes +sense to put that here, because every time you restart the game you should get +a different world. + +On the other hand, this means that the `process-input` function is no longer +pure for `:start` UIs (all the other input processing functions are still pure). +I'm not sure how I feel about that. For now I'm going to accept it, but I may +rethink it in the future. + +Now that I have the world generation in there, I can remove the `(new World)` +call from the `new-game` helper function: + +```clojure +(defn new-game [] + (new Game nil [(new UI :start)] nil)) +``` + +Now `new-game` does even less than before. It just sets up a game object with +the initial UI and `nil` world/input. + +Okay, on to the last piece: drawing the world. This is some pretty dense code, +but don't be scared — I'll guide you through it and we'll make it together: + +```clojure +(defmethod draw-ui :play [ui {{:keys [tiles]} :world :as game} screen] + (let [[cols rows] screen-size + vcols cols + vrows (dec rows) + start-x 0 + start-y 0 + end-x (+ start-x vcols) + end-y (+ start-y vrows)] + (doseq [[vrow-idx mrow-idx] (map vector + (range 0 vrows) + (range start-y end-y)) + :let [row-tiles (subvec (tiles mrow-idx) start-x end-x)]] + (doseq [vcol-idx (range vcols) + :let [{:keys [glyph color]} (row-tiles vcol-idx)]] + (s/put-string screen vcol-idx vrow-idx glyph {:fg color}))))) +``` + +Let's look at this beast line-by-line. First we pull the tile vector out of the +game object with Clojure's destructuring: + +```clojure +{{:keys [tiles]} :world :as game} +``` + +This seems a bit hard to read to me, so I might move it into the `let` statement +with a `get-in` call later. Maybe. + +Next I bind a bunch of symbols I'll be using: + +```clojure +(let [[cols rows] screen-size + vcols cols + vrows (dec rows) + start-x 0 + start-y 0 + end-x (+ start-x vcols) + end-y (+ start-y vrows)] + ,,,) +``` + +`cols` and `rows` are the dimensions of the screen (hardcoded at 80 by 24 for +the moment). + +`vcols` and `vrows` are the "viewport columns" and "viewport rows". The +"viewport" is what I'm calling the part of the screen that's actually showing +the world. I'm reserving one row at the bottom of the screen to use for +displaying the player's hit points, score, and so on. It would be trivial to +increase that to two rows if I need more later. + +`start-x` and `start-y` are the coordinates of the upper-left corner of the +viewport in the full map, and `end-x` and `end-y` are the coordinates of the +lower-right corner. For now I'm just displaying the upper-left section of the +map. In the entry after the next one I'll add the ability to scroll around. + +It's easier to explain with a diagram. Imagine I reduced the size of the world +to 10 by 10 and the terminal to 5 by 3, and the user was standing near the +middle of the map: + +```text + columns (x) + 0123456789 +rows 0.......... +(y) 1.......... + 2.......... + 3..VVVVV... + 4..VVVVV... + 5..VVVVV... + 6.......... + 7.......... + 8.......... + 9.......... +``` + +Here `V` represents the portion of the map the viewport can show (which is what +we'll be drawing to the user's terminal). In this example `start-x` would +be `2`, `start-y` would be `3`, `end-x` would be `6`, and `end-y` would be `5`. + +Okay, so I've calculated the part of the map that needs to be drawn. Now +I loop through the rows: + +```clojure +(doseq [[vrow-idx mrow-idx] (map vector + (range 0 vrows) + (range start-y end-y)) + :let [row-tiles (subvec (tiles mrow-idx) start-x end-x)]] + ,,,) +``` + +This is a bit obtuse, but basically the `map` call pairs up the viewport row and +map row indices. In our example it would result in this: + +```clojure +[[0 3] + [1 4] + [2 5]] +``` + +So viewport row `1` corresponds to map row `4`. There's probably a less +"clever" way to do this that I should use instead. + +For each row, I grab the tiles we're going to draw and store them in a vector +called `row-tiles` by grabbing the row of tiles from the world and taking +a slice of it. + +Almost there! Next I loop through each column in the row: + +```clojure +(doseq [vcol-idx (range vcols) + :let [{:keys [glyph color]} (row-tiles vcol-idx)]] + ,,,) +``` + +For each column I grab the appropriate tile and figure out what glyph and color +to draw (remember the definition of `Tile`: `(defrecord Tile [kind glyph +color])`). + +Finally I can actually draw the tile to the screen at the appropriate place: + +```clojure +(s/put-string screen vcol-idx vrow-idx glyph {:fg color}) +``` + +Whew! If that seemed painful and fiddly to you, trust me, I agree. I'm open to +suggestions on making it easier to read. + +Results +------- + +Now that the `:play` UI knows how to draw itself and process its input, and is +properly hooked up by the `:start` UI, it's time to give it a shot! + +![Screenshot](/media/images/blog/2012/07/caves-03-1-01.png) + +![Screenshot](/media/images/blog/2012/07/caves-03-1-02.png) + +![Screenshot](/media/images/blog/2012/07/caves-03-1-03.png) + +Each time we start we get a different random world. Great! + +The code is getting a bit big to include in its entirety, but you can view it +[on GitHub][result-code]. + +[result-code]: https://github.com/sjl/caves/tree/entry-03-1/src/caves + +That covers the first part of Trystan's third post. Aside from the painful +`draw-ui` function it was pretty easy to add. In the next post I'll add the +smoothing code to make the caves look a bit nicer. + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/07/caves-of-clojure-03-2.html --- a/content/blog/2012/07/caves-of-clojure-03-2.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,285 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "The Caves of Clojure: Part 3.2" - snip: "World smoothing." - created: 2012-07-10 10:04:00 - %} - -{% block article %} - -This post is part of an ongoing series. If you haven't already done so, you -should probably start at [the beginning][]. - -This entry corresponds to [post three in Trystan's tutorial][trystan-tut]. - -If you want to follow along, the code for the series is [on Bitbucket][bb] and -[on GitHub][gh]. Update to the `entry-03-2` tag to see the code as it stands -after this post. - -[the beginning]: /blog/2012/07/caves-of-clojure-01/ -[trystan-tut]: http://trystans.blogspot.com/2011/08/roguelike-tutorial-03-scrolling-through.html -[bb]: http://bitbucket.org/sjl/caves/ -[gh]: http://github.com/sjl/caves/ - -[TOC] - -Summary -------- - -When the last post left off I had a random world generated, but it wasn't very -pretty. Every tile had an equal chance of being a wall or a floor, which -results in an uninteresting "white noise" of rock. Not a very nice setting for -a roguelike. - -This post is going to show how I added Trystan's world smoothing to make -nicer-looking caves. He uses a [cellular automata-based world-smoothing -algorithm][ca-wiki] that I think is really cool, so I'm going to do pretty much -the same thing. - -Debugging ---------- - -Let's jump right in. The world smoothing code is going to go in `world.clj`. - -Before I start writing the actual smoothing code, I added two helper functions -to print worlds to the console so I could see what I was doing: - - :::clojure - (defn print-row [row] - (println (apply str (map :glyph row)))) - - (defn print-world [world] - (dorun (map print-row (:tiles world)))) - -Simple, but very helpful because `(:tiles world)` contains `Tile` records -instead of just the raw glyphs, so printing it without these helpers makes it -impossible to read. - -Smoothing ---------- - -Okay, on to the real code. I'll go through it from the bottom up this time -because I think it's easier to understand that way. - -First I added a `smooth-world` function that will be what I eventually need to -call repeatedly to smooth out the terrain: - - :::clojure - (defn smooth-world [{:keys [tiles] :as world}] - (assoc world :tiles (get-smoothed-tiles tiles))) - -It's pretty much a helper function that handles getting the tile map in and out -of the world object. The smoothing process only cares about the tile map, not -anything else the world might later contain. - -Next up: - - :::clojure - (defn get-smoothed-tiles [tiles] - (mapv (fn [y] - (get-smoothed-row tiles y)) - (range (count tiles)))) - -I use Clojure 1.4's new `mapv` function, which is basically a version of `map` -that creates a vector as the end product instead of a lazy sequence. Our -`:tiles` object is a vector of vectors going in, so it should be the same coming -out. - -I loop map over the row indices. For each row number I get the result of -`get-smoothed-row`, and the `mapv` concatenates all of those into a vector for -me, so I end up with `[[smoothed row], [smoothed row], ...]`. - -You might notice that I'm using an index-based approach here. Isn't that a bad -idea in Clojure? Shouldn't I be using sequenced-based things instead? - -I spent about twenty minutes trying to get the sequence-based approach in the -Programming Clojure book to work and eventually gave up. It sounds like -a beautiful idea but I couldn't deal with it for a number of reasons: - -* Harder to debug, with infinite padding sequences making some intermediate - steps unprintable. -* Very general, which sounds good, but makes it harder to understand because we - can't talk about "the row of tiles" any more but now talk about stuff like - "the sequence of row triples". -* In general just very alien and hard to use for what should be a - straightforward, 10 minute task. - -Here's an example of a few of the functions from the book I would have been -using if I had gone that route: - - :::clojure - (defn window - "Returns a lazy sequence of 3-item windows centered - around each item of coll, padded as necessary with - pad or nil." - ([coll] (window nil coll)) - ([pad coll] - (partition 3 1 (concat [pad] coll [pad])))) - - (defn cell-block - "Creates a sequences of 3x3 windows from a triple of 3 sequences." - [[left mid right]] - (window (map vector left mid right))) - -I personally find it easier to read things like `(get-smoothed-row tiles y)` -than `(map vector left right mid)`. You might feel differently, but this was -what I ended up with because I didn't want to spend a ton of time on the -smoothing process. - -Anyway, back to the code. Now I need a way to smooth a single row: - - :::clojure - (defn get-smoothed-row [tiles y] - (mapv (fn [x] - (get-smoothed-tile (get-block tiles x y))) - (range (count (first tiles))))) - -Once again I use `mapv` because a row needs to be a vector. This time I'm -mapping over the column indices, but for the most part it's very similar to the -previous function. - -I need a function to smooth a tile, but first I need a way to get a "block" of -tiles. - -The basic rule I'm using for the smoothing comes from the [page about cellular -automata smoothing on RogueBasin][ca-wiki]: - -[ca-wiki]: http://roguebasin.roguelikedevelopment.org/index.php?title=Cellular_Automata_Method_for_Generating_Random_Cave-Like_Levels - -A tile will become a floor tile if and only if the 3 by 3 square of tiles -centered on it contains 5 or more floor tiles. - -This means I need a way to get the 3 by 3 block of tiles centered on any given -tile: - - :::clojure - (defn block-coords [x y] - (for [dx [-1 0 1] - dy [-1 0 1]] - [(+ x dx) (+ y dy)])) - - (defn get-block [tiles x y] - (map (fn [[x y]] - (get-tile tiles x y)) - (block-coords x y))) - -First we have a helper function that returns the coordinates of all the tiles -we're going to look at. For example, if you pass it `[22 30]` it will return: - - :::clojure - [[21 29] [22 29] [23 29] - [21 30] [22 30] [23 30] - [21 31] [22 31] [23 31]] - -Note that `get-block` doesn't do any bounds checking, so passing it `[0 0]` will -happily return coordinates like `[-1 -1]`, which are off the edge of the map. - -This isn't a problem because our `get-tile` method will return `:bound` tiles -for those coordinates, which are not `:floor` tiles and so are effectively walls -for the purposes of this algorithm. - -`get-block` itself is just a glue function that gets coordinates from -`block-coords` and maps `get-tile` over them. - -So now I have a way to get a sequence of all the tiles in a block centered -around a target. The last step is actually figuring out what the resulting -block for that target should be: - - :::clojure - (defn get-smoothed-tile [block] - (let [tile-counts (frequencies (map :kind block)) - floor-threshold 5 - floor-count (get tile-counts :floor 0) - result (if (>= floor-count floor-threshold) - :floor - :wall)] - (tiles result))) - -This looks long, but that's mostly because I like using named intermediate -variables to make it more readable. It should be pretty easy to understand, -just go ahead and read through it. - -So now the `smooth-world` function has all the machinery it needs to smooth -a world. The last step is to actually *use* it. I changed the `random-world` -function to look like this: - - :::clojure - (defn random-world [] - (let [world (new World (random-tiles)) - world (nth (iterate smooth-world world) 0)] - world)) - -At the moment it takes the zeroth iteration, which actually means the unsmoothed -world. What gives? - -Interactive Development ------------------------ - -I wasn't sure right away how much smoothing would look good, so I wanted to try -out a bunch of levels and see how they behaved. I could have done it by -printing to the console, but it's a pain to compare the multiple hunks of text. - -I decided to just add it to the game itself for now to make it easy to see how -the smoothing behaves. Back in `core.clj` I pulled in the `smooth-world` -function: - - :::clojure - (ns caves.core - (:use [caves.world :only [random-world smooth-world]]) - (:require [lanterna.screen :as s])) - -Next I added another command to the `:play` UI: pressing `s` will smooth the -current world by one more level: - - - :::clojure - (defmethod process-input :play [game input] - (case input - :enter (assoc game :uis [(new UI :win)]) - :backspace (assoc game :uis [(new UI :lose)]) - \s (assoc game :world (smooth-world (:world game))) - game)) - -Yes, it only took one line to add that. I simply replace the world with the -smooth(er) world and return the resulting game. I don't need to touch the UI -stack because I want to remain at the play UI for subsequent commands. - -I'm really liking this immutable game structure so far! - -Results -------- - -Once you fire up the game and press a key to begin, you're presented with the -white-noise map from the last entry: - -![Screenshot](/media/images{{ parent_url }}/caves-03-2-01.png) - -But now you can press `s` and the caves will smooth out a bit: - -![Screenshot](/media/images{{ parent_url }}/caves-03-2-02.png) - -Another press of `s` smooths them further: - -![Screenshot](/media/images{{ parent_url }}/caves-03-2-03.png) - -You can use enter or backspace to win or lose, then any key to go back to the -start screen and get a new world to play with. - -Screenshots really don't do this justice, because seeing the world change before -your eyes is *really* cool. I made a 30-second [screencast][] that demonstrates -the effect if you don't want to actually run it locally. - -[screencast]: http://www.screenr.com/FSk8 - -I still haven't decided exactly how smooth I want to make the caves, so I'll -leave that `0` in the `nth` call for now and figure it out later. - -You can view the code [on GitHub][result-code] if you want to see it all at -once. - -[result-code]: https://github.com/sjl/caves/tree/entry-03-2/src/caves - -Next post: scrolling! - -{% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/07/caves-of-clojure-03-2.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2012/07/caves-of-clojure-03-2.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,293 @@ ++++ +title = "The Caves of Clojure: Part 3.2" +snip = "World smoothing." +date = 2012-07-10T10:04:00Z +draft = false + ++++ + +This post is part of an ongoing series. If you haven't already done so, you +should probably start at [the beginning][]. + +This entry corresponds to [post three in Trystan's tutorial][trystan-tut]. + +If you want to follow along, the code for the series is [on Bitbucket][bb] and +[on GitHub][gh]. Update to the `entry-03-2` tag to see the code as it stands +after this post. + +[the beginning]: /blog/2012/07/caves-of-clojure-01/ +[trystan-tut]: http://trystans.blogspot.com/2011/08/roguelike-tutorial-03-scrolling-through.html +[bb]: http://bitbucket.org/sjl/caves/ +[gh]: http://github.com/sjl/caves/ + +{{% toc %}} + +Summary +------- + +When the last post left off I had a random world generated, but it wasn't very +pretty. Every tile had an equal chance of being a wall or a floor, which +results in an uninteresting "white noise" of rock. Not a very nice setting for +a roguelike. + +This post is going to show how I added Trystan's world smoothing to make +nicer-looking caves. He uses a [cellular automata-based world-smoothing +algorithm][ca-wiki] that I think is really cool, so I'm going to do pretty much +the same thing. + +Debugging +--------- + +Let's jump right in. The world smoothing code is going to go in `world.clj`. + +Before I start writing the actual smoothing code, I added two helper functions +to print worlds to the console so I could see what I was doing: + +```clojure +(defn print-row [row] + (println (apply str (map :glyph row)))) + +(defn print-world [world] + (dorun (map print-row (:tiles world)))) +``` + +Simple, but very helpful because `(:tiles world)` contains `Tile` records +instead of just the raw glyphs, so printing it without these helpers makes it +impossible to read. + +Smoothing +--------- + +Okay, on to the real code. I'll go through it from the bottom up this time +because I think it's easier to understand that way. + +First I added a `smooth-world` function that will be what I eventually need to +call repeatedly to smooth out the terrain: + +```clojure +(defn smooth-world [{:keys [tiles] :as world}] + (assoc world :tiles (get-smoothed-tiles tiles))) +``` + +It's pretty much a helper function that handles getting the tile map in and out +of the world object. The smoothing process only cares about the tile map, not +anything else the world might later contain. + +Next up: + +```clojure +(defn get-smoothed-tiles [tiles] + (mapv (fn [y] + (get-smoothed-row tiles y)) + (range (count tiles)))) +``` + +I use Clojure 1.4's new `mapv` function, which is basically a version of `map` +that creates a vector as the end product instead of a lazy sequence. Our +`:tiles` object is a vector of vectors going in, so it should be the same coming +out. + +I loop map over the row indices. For each row number I get the result of +`get-smoothed-row`, and the `mapv` concatenates all of those into a vector for +me, so I end up with `[[smoothed row], [smoothed row], ...]`. + +You might notice that I'm using an index-based approach here. Isn't that a bad +idea in Clojure? Shouldn't I be using sequenced-based things instead? + +I spent about twenty minutes trying to get the sequence-based approach in the +Programming Clojure book to work and eventually gave up. It sounds like +a beautiful idea but I couldn't deal with it for a number of reasons: + +* Harder to debug, with infinite padding sequences making some intermediate + steps unprintable. +* Very general, which sounds good, but makes it harder to understand because we + can't talk about "the row of tiles" any more but now talk about stuff like + "the sequence of row triples". +* In general just very alien and hard to use for what should be a + straightforward, 10 minute task. + +Here's an example of a few of the functions from the book I would have been +using if I had gone that route: + +```clojure +(defn window + "Returns a lazy sequence of 3-item windows centered + around each item of coll, padded as necessary with + pad or nil." + ([coll] (window nil coll)) + ([pad coll] + (partition 3 1 (concat [pad] coll [pad])))) + +(defn cell-block + "Creates a sequences of 3x3 windows from a triple of 3 sequences." + [[left mid right]] + (window (map vector left mid right))) +``` + +I personally find it easier to read things like `(get-smoothed-row tiles y)` +than `(map vector left right mid)`. You might feel differently, but this was +what I ended up with because I didn't want to spend a ton of time on the +smoothing process. + +Anyway, back to the code. Now I need a way to smooth a single row: + +```clojure +(defn get-smoothed-row [tiles y] + (mapv (fn [x] + (get-smoothed-tile (get-block tiles x y))) + (range (count (first tiles))))) +``` + +Once again I use `mapv` because a row needs to be a vector. This time I'm +mapping over the column indices, but for the most part it's very similar to the +previous function. + +I need a function to smooth a tile, but first I need a way to get a "block" of +tiles. + +The basic rule I'm using for the smoothing comes from the [page about cellular +automata smoothing on RogueBasin][ca-wiki]: + +[ca-wiki]: http://roguebasin.roguelikedevelopment.org/index.php?title=Cellular_Automata_Method_for_Generating_Random_Cave-Like_Levels + +A tile will become a floor tile if and only if the 3 by 3 square of tiles +centered on it contains 5 or more floor tiles. + +This means I need a way to get the 3 by 3 block of tiles centered on any given +tile: + +```clojure +(defn block-coords [x y] + (for [dx [-1 0 1] + dy [-1 0 1]] + [(+ x dx) (+ y dy)])) + +(defn get-block [tiles x y] + (map (fn [[x y]] + (get-tile tiles x y)) + (block-coords x y))) +``` + +First we have a helper function that returns the coordinates of all the tiles +we're going to look at. For example, if you pass it `[22 30]` it will return: + +```clojure +[[21 29] [22 29] [23 29] + [21 30] [22 30] [23 30] + [21 31] [22 31] [23 31]] +``` + +Note that `get-block` doesn't do any bounds checking, so passing it `[0 0]` will +happily return coordinates like `[-1 -1]`, which are off the edge of the map. + +This isn't a problem because our `get-tile` method will return `:bound` tiles +for those coordinates, which are not `:floor` tiles and so are effectively walls +for the purposes of this algorithm. + +`get-block` itself is just a glue function that gets coordinates from +`block-coords` and maps `get-tile` over them. + +So now I have a way to get a sequence of all the tiles in a block centered +around a target. The last step is actually figuring out what the resulting +block for that target should be: + +```clojure +(defn get-smoothed-tile [block] + (let [tile-counts (frequencies (map :kind block)) + floor-threshold 5 + floor-count (get tile-counts :floor 0) + result (if (>= floor-count floor-threshold) + :floor + :wall)] + (tiles result))) +``` + +This looks long, but that's mostly because I like using named intermediate +variables to make it more readable. It should be pretty easy to understand, +just go ahead and read through it. + +So now the `smooth-world` function has all the machinery it needs to smooth +a world. The last step is to actually *use* it. I changed the `random-world` +function to look like this: + +```clojure +(defn random-world [] + (let [world (new World (random-tiles)) + world (nth (iterate smooth-world world) 0)] + world)) +``` + +At the moment it takes the zeroth iteration, which actually means the unsmoothed +world. What gives? + +Interactive Development +----------------------- + +I wasn't sure right away how much smoothing would look good, so I wanted to try +out a bunch of levels and see how they behaved. I could have done it by +printing to the console, but it's a pain to compare the multiple hunks of text. + +I decided to just add it to the game itself for now to make it easy to see how +the smoothing behaves. Back in `core.clj` I pulled in the `smooth-world` +function: + +```clojure +(ns caves.core + (:use [caves.world :only [random-world smooth-world]]) + (:require [lanterna.screen :as s])) +``` + +Next I added another command to the `:play` UI: pressing `s` will smooth the +current world by one more level: + + +```clojure +(defmethod process-input :play [game input] + (case input + :enter (assoc game :uis [(new UI :win)]) + :backspace (assoc game :uis [(new UI :lose)]) + \s (assoc game :world (smooth-world (:world game))) + game)) +``` + +Yes, it only took one line to add that. I simply replace the world with the +smooth(er) world and return the resulting game. I don't need to touch the UI +stack because I want to remain at the play UI for subsequent commands. + +I'm really liking this immutable game structure so far! + +Results +------- + +Once you fire up the game and press a key to begin, you're presented with the +white-noise map from the last entry: + +![Screenshot](/media/images/blog/2012/07/caves-03-2-01.png) + +But now you can press `s` and the caves will smooth out a bit: + +![Screenshot](/media/images/blog/2012/07/caves-03-2-02.png) + +Another press of `s` smooths them further: + +![Screenshot](/media/images/blog/2012/07/caves-03-2-03.png) + +You can use enter or backspace to win or lose, then any key to go back to the +start screen and get a new world to play with. + +Screenshots really don't do this justice, because seeing the world change before +your eyes is *really* cool. I made a 30-second [screencast][] that demonstrates +the effect if you don't want to actually run it locally. + +[screencast]: http://www.screenr.com/FSk8 + +I still haven't decided exactly how smooth I want to make the caves, so I'll +leave that `0` in the `nth` call for now and figure it out later. + +You can view the code [on GitHub][result-code] if you want to see it all at +once. + +[result-code]: https://github.com/sjl/caves/tree/entry-03-2/src/caves + +Next post: scrolling! + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/07/caves-of-clojure-03-3.html --- a/content/blog/2012/07/caves-of-clojure-03-3.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,399 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "The Caves of Clojure: Part 3.3" - snip: "Scrolling." - created: 2012-07-11 9:25:00 - %} - -{% block article %} - -This post is part of an ongoing series. If you haven't already done so, you -should probably start at [the beginning][]. - -This entry corresponds to [post three in Trystan's tutorial][trystan-tut]. - -If you want to follow along, the code for the series is [on Bitbucket][bb] and -[on GitHub][gh]. Update to the `entry-03-3` tag to see the code as it stands -after this post. - -[the beginning]: /blog/2012/07/caves-of-clojure-01/ -[trystan-tut]: http://trystans.blogspot.com/2011/08/roguelike-tutorial-03-scrolling-through.html -[bb]: http://bitbucket.org/sjl/caves/ -[gh]: http://github.com/sjl/caves/ - -[TOC] - -Summary -------- - -When the last post left off I had a random world generated and smoothed to -create some nice looking caves. The world was displayed on the screen, but it -would only display the upper left corner of the map. - -This post is going to be about scrolling the viewport so we can view the entire -map. It's the last remaining piece of Trystan's third post that I still need to -implement. - -Refactoring ------------ - -This is going to involve changing the worst function in the code so far -(`draw-ui` for `:player` UIs), so before I start hacking away I want to factor -out a bit of functionality to clean things up. - -Right now that `draw-ui` function in `core.clj` looks like this: - - :::clojure - (defmethod draw-ui :play [ui {{:keys [tiles]} :world :as game} screen] - (let [[cols rows] screen-size - vcols cols - vrows (dec rows) - start-x 0 - start-y 0 - end-x (+ start-x vcols) - end-y (+ start-y vrows)] - (doseq [[vrow-idx mrow-idx] (map vector - (range 0 vrows) - (range start-y end-y)) - :let [row-tiles (subvec (tiles mrow-idx) start-x end-x)]] - (doseq [vcol-idx (range vcols) - :let [{:keys [glyph color]} (row-tiles vcol-idx)]] - (s/put-string screen vcol-idx vrow-idx glyph {:fg color}))))) - -I pulled out the guts of that function into a helper function: - - :::clojure - (defn draw-world [screen vrows vcols start-x start-y end-x end-y tiles] - (doseq [[vrow-idx mrow-idx] (map vector - (range 0 vrows) - (range start-y end-y)) - :let [row-tiles (subvec (tiles mrow-idx) start-x end-x)]] - (doseq [vcol-idx (range vcols) - :let [{:keys [glyph color]} (row-tiles vcol-idx)]] - (s/put-string screen vcol-idx vrow-idx glyph {:fg color})))) - - (defmethod draw-ui :play [ui {{:keys [tiles]} :world :as game} screen] - (let [[cols rows] screen-size - vcols cols - vrows (dec rows) - start-x 0 - start-y 0 - end-x (+ start-x vcols) - end-y (+ start-y vrows)] - (draw-world screen vrows vcols start-x start-y end-x end-y tiles))) - -No functionality has changed, I just pulled the body out into its own function. -This will make things cleaner as we add more functionality. - -As I mentioned in the last post, I don't like the distructuring in the argument -list here. Let's remove that: - - :::clojure - (defmethod draw-ui :play [ui game screen] - (let [world (:world game) - tiles (:tiles world) - [cols rows] screen-size - vcols cols - vrows (dec rows) - start-x 0 - start-y 0 - end-x (+ start-x vcols) - end-y (+ start-y vrows)] - (draw-world screen vrows vcols start-x start-y end-x end-y tiles))) - -It's a few more lines of code but I find it more readable. If you prefer the -more concise syntax feel free to use the destructuring -- it's not really that -important either way. - -Crosshairs ----------- - -Trystan draws an `X` as a kind of crosshair to take the place of the traditional -roguelike `@` (since there's no player yet), so let's do that. I made -a separate function to draw the crosshair as a red `X` in the center of the -screen: - - :::clojure - (defn draw-crosshairs [screen vcols vrows] - (let [crosshair-x (int (/ vcols 2)) - crosshair-y (int (/ vrows 2))] - (s/put-string screen crosshair-x crosshair-y "X" {:fg :red}) - (s/move-cursor screen crosshair-x crosshair-y))) - -This function seems pretty straightforward. It finds the x and y coordinates of -the viewport where the `X` should go and puts it there. It also moves the -cursor on top of it because I like how that looks. - -Yeah, it might not actually end up in the exact center of the screen because the -`int` will truncate if we've got an odd number of rows or columns. Honestly, -I'm going to be throwing away this crosshair once we've got a player on the -screen, so it's not worth fixing. - -I need to call `draw-crosshairs` that in the `:play` UI-drawing function: - - :::clojure - (defmethod draw-ui :play [ui game screen] - (let [world (:world game) - tiles (:tiles world) - [cols rows] screen-size - vcols cols - vrows (dec rows) - start-x 0 - start-y 0 - end-x (+ start-x vcols) - end-y (+ start-y vrows)] - (draw-world screen vrows vcols start-x start-y end-x end-y tiles) - (draw-crosshairs screen vcols vrows))) - -The only change here is the `(draw-crosshairs screen vcols vrows)` after I draw -the world. This draws the crosshair `X` on top of the world, which isn't an -issue because Lanterna's double buffering will ensure that the user never sees -an intermediate render that's missing the `X`. - -Now there's a red `X` in the center of the screen. Great, but we still need to -add the main point of this post: scrolling. - -Scrolling ---------- - -Right now the `start-x` and `start-y` in the `draw-ui` function are hardcoded at -`0`. All I need to do is change those to modify which part of the map the -viewport draws, and I'll have scrolling! - -First of all, I need a way to keep track of where the viewport should be -centered. This will get thrown away once we have a player (the player will be -the center of the viewport), so I'll just slap it right in the `game` object for -now: - - :::clojure - (defn new-game [] - (assoc (new Game nil [(new UI :start)] nil) - :location [40 20])) - -The `new-game` function now `assoc`s a `:location` into the `game` before -returning it. - -I *could* have modified the `(defrecord Game [world uis input])` to add the -location as a proper field. But I know I'm going to be removing this soon -anyway, so I may as well take advantage of the fact that Clojure's record can -have extra fields `assoc`ed onto them on the fly. - -`[40 20]` is an arbitrary location. It's kind of in the middleish area of the -map. Good enough. - -Okay, now I need to actually display the correct area of the map in the -viewport. I'm going to need to modify `draw-ui` again, which, just as -a reminder, looks like this: - - :::clojure - (defmethod draw-ui :play [ui game screen] - (let [world (:world game) - tiles (:tiles world) - [cols rows] screen-size - vcols cols - vrows (dec rows) - start-x 0 - start-y 0 - end-x (+ start-x vcols) - end-y (+ start-y vrows)] - (draw-world screen vrows vcols start-x start-y end-x end-y tiles) - (draw-crosshairs screen vcols vrows))) - -I had a feeling this is going to get a bit gross, so I pulled out the code for -getting the viewport coordinates into its own helper function: - - :::clojure - (defn get-viewport-coords [game vcols vrows] - (let [start-x 0 - start-y 0 - end-x (+ start-x vcols) - end-y (+ start-y vrows)]] - [start-x start-y end-x end-y])) - - (defmethod draw-ui :play [ui game screen] - (let [world (:world game) - tiles (:tiles world) - [cols rows] screen-size - vcols cols - vrows (dec rows) - [start-x start-y end-x end-y] (get-viewport-coords game vcols vrows)]] - (draw-world screen vrows vcols start-x start-y end-x end-y tiles) - (draw-crosshairs screen vcols vrows))) - -No functionality changed, I just shuffled a bit of code out of that ugly -`draw-ui` function. As a bonus, the `get-viewport-coords` function is now pure. -It'll be easy to add unit tests for it later if I want. Cool. - -Now that the viewport coordinates are isolated, it's time to calculate them -correctly instead of hardcoding them at `0`: - - :::clojure - (defn get-viewport-coords [game vcols vrows] - (let [location (:location game) - [center-x center-y] location - - tiles (:tiles (:world game)) - - map-rows (count tiles) - map-cols (count (first tiles)) - - start-x (max 0 (- center-x (int (/ vcols 2)))) - start-y (max 0 (- center-y (int (/ vrows 2)))) - - end-x (+ start-x vcols) - end-x (min end-x map-cols) - - end-y (+ start-y vrows) - end-y (min end-y map-rows) - - start-x (- end-x vcols) - start-y (- end-y vrows)] - [start-x start-y end-x end-y])) - -This is long, but very straightforward. I use the fact that `let` doesn't care -if you rebind variables many times in the same binding vector to write -imperative code. There may be a more "clever" way to do this, but I like the -clarity. - -First it finds the location of the crosshair (which will be `[40 20]` from -`new-game` at the moment). It calls that `center-x` and `center-y`. - -It also pulls the tile vector out of the `game` object and uses it to -determine the full dimensions of the map. I'm thinking of having a `map-size` -constant somewhere instead of doing it this way. I may do that in a later post. - -Next come these scary lines: - - :::clojure - start-x (max 0 (- center-x (int (/ vcols 2)))) - start-y (max 0 (- center-y (int (/ vrows 2)))) - -They're not as scary as they look. Both are exactly the same except for which -dimension they're working on. First I subtract half the viewport size from the -center coordinate. This should give me either the topmost or leftmost -coordinate we're going to be drawing. - -Then I use `max` to make sure that if the starting coordinate would be less than -zero (i.e.: off of the map) I just use 0 instead. - -Okay, so now I've got the coordinates of the top left point I need to draw, and -I'm sure that it doesn't fall off the top or left edge of the map. Cool. Time -to get the bottom right coordinate. - - :::clojure - end-x (+ start-x vcols) - end-x (min end-x map-cols) - - end-y (+ start-y vrows) - end-y (min end-y map-rows) - -This is similar to how we get the starting coordinates. We calculate a "naive -end x" by adding the viewport size to the start, and then make sure the end -doesn't fall off the map. - -I did this all in one line for the start coordinates, but split it into two for -the end coordinates. I'm not sure why I did it like that -- I just noticed it -now. I'm going to go ahead and change the start to be the expanded, two-line -form. I think it's clearer. - -Okay, so now I've ensured that the end coordinate doesn't fall off the map. I'm -done, right? - -Well, not quite. If I truncated the end coordinate here I'll have ended up with -a smaller-than-normal viewport. To fix that I'll reset the start coordinates -one more time: - - :::clojure - start-x (- end-x vcols) - start-y (- end-y vrows) - -This time I don't need to check any bounds. I know the end coordinate is good -because it was based on a known start coordinate (the top/left side is good) and -I corrected the bottom/right side. So I simply use this known-good end -coordinate to get a known-good start coordinate and I'm done. - -If the map is smaller than the viewport size this is probably going to explode. -I'm going to ignore that for now. I may revisit it later, or I may just stick -with an 80 by 24 viewport for all time like Nethack. - -That was a lot of work, but the only thing that's changed is I'm now displaying -a section of the map near the middle instead of at the upper left. The last -piece is to add the ability to adjust the `:location` in the `game` object on -the fly. - -The player should be able to scroll around when they're at the `:play` UI, so -let's add the appropriate input handling: - - :::clojure - (defn move [[x y] [dx dy]] - [(+ x dx) (+ y dy)]) - - (defmethod process-input :play [game input] - (case input - :enter (assoc game :uis [(new UI :win)]) - :backspace (assoc game :uis [(new UI :lose)]) - \q (assoc game :uis []) - - \s (assoc game :world (smooth-world (:world game))) - - \h (update-in game [:location] move [-1 0]) - \j (update-in game [:location] move [0 1]) - \k (update-in game [:location] move [0 -1]) - \l (update-in game [:location] move [1 0]) - - \H (update-in game [:location] move [-5 0]) - \J (update-in game [:location] move [0 5]) - \K (update-in game [:location] move [0 -5]) - \L (update-in game [:location] move [5 0]) - - game)) - -I did a few things here. First I added the `q` key mapping to quit the game -without going through the win or lose screens, just to same myself some time. -Enter and backspace still win and lose the game respectively. - -`s` still smooths the world map for now. No reason to remove that yet. - -To handle the movement inputs I first made a `move` helper function which takes -a coordinate and an amount to move by and returns the new coordinate. - -The `process-input` function uses this to get the new coordiate when it gets an -`h`, `j`, `k`, or `l` keypress. I also added the shifted versions of the -letters as "fast movement" keys for convenience. - -Right now there's no bounds checking here, so it's possible for your `:location` -to get scrolled off the edge of the map. This won't be a problem for the -display (it will just snap the viewport to the edge of the map), but will make -the input a bit weird. - -For example, if you scroll to the right edge of the map and press right 10 more -times, you'll need to press left 10 times before it will actually start -scrolling left again. - -This is a bug, but not one I care to fix right now. I'll be replacing this code -with player-based code soon enough, so it's just going to get thrown out anyway. - -Results -------- - -That's it! Running the game, I can now scroll around the map and/or smooth it -whenever I like: - -![Screenshot](/media/images{{ parent_url }}/caves-03-3-01.png) - -![Screenshot](/media/images{{ parent_url }}/caves-03-3-02.png) - -This doesn't look much different in pictures, but I can scroll through the world -with `hjkl`. Here's a screencast showing what that looks like: - - -As always, you can view the code [on GitHub][result-code] if you want to see it -all at once. - -[result-code]: https://github.com/sjl/caves/tree/entry-03-3/src/caves - -That's it for Trystan's third post. Next time I'll tackle his fourth (adding -an actual player). - -{% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/07/caves-of-clojure-03-3.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2012/07/caves-of-clojure-03-3.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,409 @@ ++++ +title = "The Caves of Clojure: Part 3.3" +snip = "Scrolling." +date = 2012-07-11T9:25:00Z +draft = false + ++++ + +This post is part of an ongoing series. If you haven't already done so, you +should probably start at [the beginning][]. + +This entry corresponds to [post three in Trystan's tutorial][trystan-tut]. + +If you want to follow along, the code for the series is [on Bitbucket][bb] and +[on GitHub][gh]. Update to the `entry-03-3` tag to see the code as it stands +after this post. + +[the beginning]: /blog/2012/07/caves-of-clojure-01/ +[trystan-tut]: http://trystans.blogspot.com/2011/08/roguelike-tutorial-03-scrolling-through.html +[bb]: http://bitbucket.org/sjl/caves/ +[gh]: http://github.com/sjl/caves/ + +{{% toc %}} + +Summary +------- + +When the last post left off I had a random world generated and smoothed to +create some nice looking caves. The world was displayed on the screen, but it +would only display the upper left corner of the map. + +This post is going to be about scrolling the viewport so we can view the entire +map. It's the last remaining piece of Trystan's third post that I still need to +implement. + +Refactoring +----------- + +This is going to involve changing the worst function in the code so far +(`draw-ui` for `:player` UIs), so before I start hacking away I want to factor +out a bit of functionality to clean things up. + +Right now that `draw-ui` function in `core.clj` looks like this: + +```clojure +(defmethod draw-ui :play [ui {{:keys [tiles]} :world :as game} screen] + (let [[cols rows] screen-size + vcols cols + vrows (dec rows) + start-x 0 + start-y 0 + end-x (+ start-x vcols) + end-y (+ start-y vrows)] + (doseq [[vrow-idx mrow-idx] (map vector + (range 0 vrows) + (range start-y end-y)) + :let [row-tiles (subvec (tiles mrow-idx) start-x end-x)]] + (doseq [vcol-idx (range vcols) + :let [{:keys [glyph color]} (row-tiles vcol-idx)]] + (s/put-string screen vcol-idx vrow-idx glyph {:fg color}))))) +``` + +I pulled out the guts of that function into a helper function: + +```clojure +(defn draw-world [screen vrows vcols start-x start-y end-x end-y tiles] + (doseq [[vrow-idx mrow-idx] (map vector + (range 0 vrows) + (range start-y end-y)) + :let [row-tiles (subvec (tiles mrow-idx) start-x end-x)]] + (doseq [vcol-idx (range vcols) + :let [{:keys [glyph color]} (row-tiles vcol-idx)]] + (s/put-string screen vcol-idx vrow-idx glyph {:fg color})))) + +(defmethod draw-ui :play [ui {{:keys [tiles]} :world :as game} screen] + (let [[cols rows] screen-size + vcols cols + vrows (dec rows) + start-x 0 + start-y 0 + end-x (+ start-x vcols) + end-y (+ start-y vrows)] + (draw-world screen vrows vcols start-x start-y end-x end-y tiles))) +``` + +No functionality has changed, I just pulled the body out into its own function. +This will make things cleaner as we add more functionality. + +As I mentioned in the last post, I don't like the distructuring in the argument +list here. Let's remove that: + +```clojure +(defmethod draw-ui :play [ui game screen] + (let [world (:world game) + tiles (:tiles world) + [cols rows] screen-size + vcols cols + vrows (dec rows) + start-x 0 + start-y 0 + end-x (+ start-x vcols) + end-y (+ start-y vrows)] + (draw-world screen vrows vcols start-x start-y end-x end-y tiles))) +``` + +It's a few more lines of code but I find it more readable. If you prefer the +more concise syntax feel free to use the destructuring — it's not really that +important either way. + +Crosshairs +---------- + +Trystan draws an `X` as a kind of crosshair to take the place of the traditional +roguelike `@` (since there's no player yet), so let's do that. I made +a separate function to draw the crosshair as a red `X` in the center of the +screen: + +```clojure +(defn draw-crosshairs [screen vcols vrows] + (let [crosshair-x (int (/ vcols 2)) + crosshair-y (int (/ vrows 2))] + (s/put-string screen crosshair-x crosshair-y "X" {:fg :red}) + (s/move-cursor screen crosshair-x crosshair-y))) +``` + +This function seems pretty straightforward. It finds the x and y coordinates of +the viewport where the `X` should go and puts it there. It also moves the +cursor on top of it because I like how that looks. + +Yeah, it might not actually end up in the exact center of the screen because the +`int` will truncate if we've got an odd number of rows or columns. Honestly, +I'm going to be throwing away this crosshair once we've got a player on the +screen, so it's not worth fixing. + +I need to call `draw-crosshairs` that in the `:play` UI-drawing function: + +```clojure +(defmethod draw-ui :play [ui game screen] + (let [world (:world game) + tiles (:tiles world) + [cols rows] screen-size + vcols cols + vrows (dec rows) + start-x 0 + start-y 0 + end-x (+ start-x vcols) + end-y (+ start-y vrows)] + (draw-world screen vrows vcols start-x start-y end-x end-y tiles) + (draw-crosshairs screen vcols vrows))) +``` + +The only change here is the `(draw-crosshairs screen vcols vrows)` after I draw +the world. This draws the crosshair `X` on top of the world, which isn't an +issue because Lanterna's double buffering will ensure that the user never sees +an intermediate render that's missing the `X`. + +Now there's a red `X` in the center of the screen. Great, but we still need to +add the main point of this post: scrolling. + +Scrolling +--------- + +Right now the `start-x` and `start-y` in the `draw-ui` function are hardcoded at +`0`. All I need to do is change those to modify which part of the map the +viewport draws, and I'll have scrolling! + +First of all, I need a way to keep track of where the viewport should be +centered. This will get thrown away once we have a player (the player will be +the center of the viewport), so I'll just slap it right in the `game` object for +now: + +```clojure +(defn new-game [] + (assoc (new Game nil [(new UI :start)] nil) + :location [40 20])) +``` + +The `new-game` function now `assoc`s a `:location` into the `game` before +returning it. + +I *could* have modified the `(defrecord Game [world uis input])` to add the +location as a proper field. But I know I'm going to be removing this soon +anyway, so I may as well take advantage of the fact that Clojure's record can +have extra fields `assoc`ed onto them on the fly. + +`[40 20]` is an arbitrary location. It's kind of in the middleish area of the +map. Good enough. + +Okay, now I need to actually display the correct area of the map in the +viewport. I'm going to need to modify `draw-ui` again, which, just as +a reminder, looks like this: + +```clojure +(defmethod draw-ui :play [ui game screen] + (let [world (:world game) + tiles (:tiles world) + [cols rows] screen-size + vcols cols + vrows (dec rows) + start-x 0 + start-y 0 + end-x (+ start-x vcols) + end-y (+ start-y vrows)] + (draw-world screen vrows vcols start-x start-y end-x end-y tiles) + (draw-crosshairs screen vcols vrows))) +``` + +I had a feeling this is going to get a bit gross, so I pulled out the code for +getting the viewport coordinates into its own helper function: + +```clojure +(defn get-viewport-coords [game vcols vrows] + (let [start-x 0 + start-y 0 + end-x (+ start-x vcols) + end-y (+ start-y vrows)]] + [start-x start-y end-x end-y])) + +(defmethod draw-ui :play [ui game screen] + (let [world (:world game) + tiles (:tiles world) + [cols rows] screen-size + vcols cols + vrows (dec rows) + [start-x start-y end-x end-y] (get-viewport-coords game vcols vrows)]] + (draw-world screen vrows vcols start-x start-y end-x end-y tiles) + (draw-crosshairs screen vcols vrows))) +``` + +No functionality changed, I just shuffled a bit of code out of that ugly +`draw-ui` function. As a bonus, the `get-viewport-coords` function is now pure. +It'll be easy to add unit tests for it later if I want. Cool. + +Now that the viewport coordinates are isolated, it's time to calculate them +correctly instead of hardcoding them at `0`: + +```clojure +(defn get-viewport-coords [game vcols vrows] + (let [location (:location game) + [center-x center-y] location + + tiles (:tiles (:world game)) + + map-rows (count tiles) + map-cols (count (first tiles)) + + start-x (max 0 (- center-x (int (/ vcols 2)))) + start-y (max 0 (- center-y (int (/ vrows 2)))) + + end-x (+ start-x vcols) + end-x (min end-x map-cols) + + end-y (+ start-y vrows) + end-y (min end-y map-rows) + + start-x (- end-x vcols) + start-y (- end-y vrows)] + [start-x start-y end-x end-y])) +``` + +This is long, but very straightforward. I use the fact that `let` doesn't care +if you rebind variables many times in the same binding vector to write +imperative code. There may be a more "clever" way to do this, but I like the +clarity. + +First it finds the location of the crosshair (which will be `[40 20]` from +`new-game` at the moment). It calls that `center-x` and `center-y`. + +It also pulls the tile vector out of the `game` object and uses it to +determine the full dimensions of the map. I'm thinking of having a `map-size` +constant somewhere instead of doing it this way. I may do that in a later post. + +Next come these scary lines: + +```clojure +start-x (max 0 (- center-x (int (/ vcols 2)))) +start-y (max 0 (- center-y (int (/ vrows 2)))) +``` + +They're not as scary as they look. Both are exactly the same except for which +dimension they're working on. First I subtract half the viewport size from the +center coordinate. This should give me either the topmost or leftmost +coordinate we're going to be drawing. + +Then I use `max` to make sure that if the starting coordinate would be less than +zero (i.e.: off of the map) I just use 0 instead. + +Okay, so now I've got the coordinates of the top left point I need to draw, and +I'm sure that it doesn't fall off the top or left edge of the map. Cool. Time +to get the bottom right coordinate. + +```clojure +end-x (+ start-x vcols) +end-x (min end-x map-cols) + +end-y (+ start-y vrows) +end-y (min end-y map-rows) +``` + +This is similar to how we get the starting coordinates. We calculate a "naive +end x" by adding the viewport size to the start, and then make sure the end +doesn't fall off the map. + +I did this all in one line for the start coordinates, but split it into two for +the end coordinates. I'm not sure why I did it like that — I just noticed it +now. I'm going to go ahead and change the start to be the expanded, two-line +form. I think it's clearer. + +Okay, so now I've ensured that the end coordinate doesn't fall off the map. I'm +done, right? + +Well, not quite. If I truncated the end coordinate here I'll have ended up with +a smaller-than-normal viewport. To fix that I'll reset the start coordinates +one more time: + +```clojure +start-x (- end-x vcols) +start-y (- end-y vrows) +``` + +This time I don't need to check any bounds. I know the end coordinate is good +because it was based on a known start coordinate (the top/left side is good) and +I corrected the bottom/right side. So I simply use this known-good end +coordinate to get a known-good start coordinate and I'm done. + +If the map is smaller than the viewport size this is probably going to explode. +I'm going to ignore that for now. I may revisit it later, or I may just stick +with an 80 by 24 viewport for all time like Nethack. + +That was a lot of work, but the only thing that's changed is I'm now displaying +a section of the map near the middle instead of at the upper left. The last +piece is to add the ability to adjust the `:location` in the `game` object on +the fly. + +The player should be able to scroll around when they're at the `:play` UI, so +let's add the appropriate input handling: + +```clojure +(defn move [[x y] [dx dy]] + [(+ x dx) (+ y dy)]) + +(defmethod process-input :play [game input] + (case input + :enter (assoc game :uis [(new UI :win)]) + :backspace (assoc game :uis [(new UI :lose)]) + \q (assoc game :uis []) + + \s (assoc game :world (smooth-world (:world game))) + + \h (update-in game [:location] move [-1 0]) + \j (update-in game [:location] move [0 1]) + \k (update-in game [:location] move [0 -1]) + \l (update-in game [:location] move [1 0]) + + \H (update-in game [:location] move [-5 0]) + \J (update-in game [:location] move [0 5]) + \K (update-in game [:location] move [0 -5]) + \L (update-in game [:location] move [5 0]) + + game)) +``` + +I did a few things here. First I added the `q` key mapping to quit the game +without going through the win or lose screens, just to same myself some time. +Enter and backspace still win and lose the game respectively. + +`s` still smooths the world map for now. No reason to remove that yet. + +To handle the movement inputs I first made a `move` helper function which takes +a coordinate and an amount to move by and returns the new coordinate. + +The `process-input` function uses this to get the new coordiate when it gets an +`h`, `j`, `k`, or `l` keypress. I also added the shifted versions of the +letters as "fast movement" keys for convenience. + +Right now there's no bounds checking here, so it's possible for your `:location` +to get scrolled off the edge of the map. This won't be a problem for the +display (it will just snap the viewport to the edge of the map), but will make +the input a bit weird. + +For example, if you scroll to the right edge of the map and press right 10 more +times, you'll need to press left 10 times before it will actually start +scrolling left again. + +This is a bug, but not one I care to fix right now. I'll be replacing this code +with player-based code soon enough, so it's just going to get thrown out anyway. + +Results +------- + +That's it! Running the game, I can now scroll around the map and/or smooth it +whenever I like: + +![Screenshot](/media/images/blog/2012/07/caves-03-3-01.png) + +![Screenshot](/media/images/blog/2012/07/caves-03-3-02.png) + +This doesn't look much different in pictures, but I can scroll through the world +with `hjkl`. Here's a screencast showing what that looks like: + + +As always, you can view the code [on GitHub][result-code] if you want to see it +all at once. + +[result-code]: https://github.com/sjl/caves/tree/entry-03-3/src/caves + +That's it for Trystan's third post. Next time I'll tackle his fourth (adding +an actual player). + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/07/caves-of-clojure-03-4.html --- a/content/blog/2012/07/caves-of-clojure-03-4.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,336 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "The Caves of Clojure: Part 3.4" - snip: "Refactoring." - created: 2012-07-11 12:02:00 - %} - -{% block article %} - -This post is part of an ongoing series. If you haven't already done so, you -should probably start at [the beginning][]. - -This entry (kind of) corresponds to [post three in Trystan's -tutorial][trystan-tut]. - -If you want to follow along, the code for the series is [on Bitbucket][bb] and -[on GitHub][gh]. Update to the `entry-03-4` tag to see the code as it stands -after this post. - -[the beginning]: /blog/2012/07/caves-of-clojure-01/ -[trystan-tut]: http://trystans.blogspot.com/2011/08/roguelike-tutorial-03-scrolling-through.html -[bb]: http://bitbucket.org/sjl/caves/ -[gh]: http://github.com/sjl/caves/ - -[TOC] - -Summary -------- - -In the last post I said that the next post would be about Trystan's fourth -entry. I lied. I'm going to do a short entry about refactoring before I move -on, because I don't want to clutter up later ones with this stuff. - -Record Creation ---------------- - -Hacker News user bitsai [told me about][hn] a new syntax for creating records in -Clojure 1.3, so I updated all the `(new Foo)` calls to use that. - -One example is the `new-game` function. Before: - -[hn]: https://news.ycombinator.com/item?id=4220141 - - :::clojure - (defn new-game [] - (assoc (new Game nil [(new UI :start)] nil) - :location [40 20])) - -After: - - :::clojure - (defn new-game [] - (assoc (->Game nil [(->UI :start)] nil) - :location [40 20])) - -It's certainly not too impressive from a "characters saved" point of view. But -this little change matters more than it might appear at first glance. - -Imagine you define a record in Clojure: - - :::clojure - (ns a) - - (defrecord Foo []) - -And then in another namespace you want to use it: - - :::clojure - (ns b - (:use a)) - - (new Foo) - -This will explode, because `Foo` is actually a Java class, so you need to import -it: - - :::clojure - (ns b - (:import a.Foo) - (:use a)) - - (new Foo) - -This is one example of Clojure's Java underpinnings leaking through. - -In Clojure 1.3, `defrecord` will automatically generate a "factory function" -that creates the record, and you can `require` or `use` *that* like any other -function, so you don't need to screw around with a Java interop feature -(`import`) to use a pure Clojure feature. - -This is a good thing. It means that progress is being made toward patching the -places that Java leaks into Clojure. It gives me hope that some day I'll feel -okay recommending Clojure to people without Java experience. - -I updated all the `(new ...)` calls to use the new-style factory functions. -I won't paste them all here, but if you're following along you'll want to `grep --R 'new ' .` and update the rest now. - -update-in ---------- - -Next is a tiny change that's just a bit cleaner. Alan Malloy [told me][] about -it. In the `process-input` function for the `:play` UI, the code to handle -smoothing the world looked like this: - -[told me]: https://twitter.com/alanmalloy/status/222748536595423232 - - :::clojure - \s (assoc game :world (smooth-world (:world game))) - -This can be done much more cleanly using `update-in`: - - :::clojure - \s (update-in game [:world] smooth-world) - -Nice. - -Namespaces ----------- - -I said in an earlier post that I tend to leave things in one file until I feel -like they need to be pulled out. Well, that time has come. - -First I pulled the UI drawing code into its own file: `ui/drawing.clj`. It -looks like this (nothing has changed, it's just in a file of its own now): - - :::clojure - (ns caves.ui.drawing - (:require [lanterna.screen :as s])) - - - (def screen-size [80 24]) - - (defn clear-screen [screen] - (let [[cols rows] screen-size - blank (apply str (repeat cols \space))] - (doseq [row (range rows)] - (s/put-string screen 0 row blank)))) - - - (defmulti draw-ui - (fn [ui game screen] - (:kind ui))) - - - (defmethod draw-ui :start [ui game screen] - (s/put-string screen 0 0 "Welcome to the Caves of Clojure!") - (s/put-string screen 0 1 "Press any key to continue.") - (s/put-string screen 0 2 "") - (s/put-string screen 0 3 "Once in the game, you can use enter to win,") - (s/put-string screen 0 4 "and backspace to lose.")) - - - (defmethod draw-ui :win [ui game screen] - (s/put-string screen 0 0 "Congratulations, you win!") - (s/put-string screen 0 1 "Press escape to exit, anything else to restart.")) - - - (defmethod draw-ui :lose [ui game screen] - (s/put-string screen 0 0 "Sorry, better luck next time.") - (s/put-string screen 0 1 "Press escape to exit, anything else to restart.")) - - - (defn get-viewport-coords [game vcols vrows] - (let [location (:location game) - [center-x center-y] location - - tiles (:tiles (:world game)) - - map-rows (count tiles) - map-cols (count (first tiles)) - - start-x (- center-x (int (/ vcols 2))) - start-x (max 0 start-x) - - start-y (- center-y (int (/ vrows 2))) - start-y (max 0 start-y) - - end-x (+ start-x vcols) - end-x (min end-x map-cols) - - end-y (+ start-y vrows) - end-y (min end-y map-rows) - - start-x (- end-x vcols) - start-y (- end-y vrows)] - [start-x start-y end-x end-y])) - - (defn draw-crosshairs [screen vcols vrows] - (let [crosshair-x (int (/ vcols 2)) - crosshair-y (int (/ vrows 2))] - (s/put-string screen crosshair-x crosshair-y "X" {:fg :red}) - (s/move-cursor screen crosshair-x crosshair-y))) - - (defn draw-world [screen vrows vcols start-x start-y end-x end-y tiles] - (doseq [[vrow-idx mrow-idx] (map vector - (range 0 vrows) - (range start-y end-y)) - :let [row-tiles (subvec (tiles mrow-idx) start-x end-x)]] - (doseq [vcol-idx (range vcols) - :let [{:keys [glyph color]} (row-tiles vcol-idx)]] - (s/put-string screen vcol-idx vrow-idx glyph {:fg color})))) - - (defmethod draw-ui :play [ui game screen] - (let [world (:world game) - tiles (:tiles world) - [cols rows] screen-size - vcols cols - vrows (dec rows) - [start-x start-y end-x end-y] (get-viewport-coords game vcols vrows)] - (draw-world screen vrows vcols start-x start-y end-x end-y tiles) - (draw-crosshairs screen vcols vrows))) - - - (defn draw-game [game screen] - (clear-screen screen) - (doseq [ui (:uis game)] - (draw-ui ui game screen)) - (s/redraw screen)) - -And now I need to `use` the `draw-game` function back in `core.clj`: - - :::clojure - (ns caves.core - (:use [caves.world :only [random-world smooth-world]] - [caves.ui.drawing :only [draw-game]]) - (:require [lanterna.screen :as s])) - -The fact that I moved eleven top-level symbols into a new namespace and only had -to bring one of them back into the original is a pretty clear sign that this -chunk of code was ready to be moved into its own file. - -I also did the same for the input processing code, moving it into -`ui/input.clj`: - - :::clojure - (ns caves.ui.input - (:use [caves.world :only [random-world smooth-world]]) - (:require [lanterna.screen :as s])) - - - (defmulti process-input - (fn [game input] - (:kind (last (:uis game))))) - - (defmethod process-input :start [game input] - (-> game - (assoc :world (random-world)) - (assoc :uis [(->UI :play)]))) - - - (defn move [[x y] [dx dy]] - [(+ x dx) (+ y dy)]) - - (defmethod process-input :play [game input] - (case input - :enter (assoc game :uis [(->UI :win)]) - :backspace (assoc game :uis [(->UI :lose)]) - \q (assoc game :uis []) - - \s (update-in game [:world] smooth-world) - - \h (update-in game [:location] move [-1 0]) - \j (update-in game [:location] move [0 1]) - \k (update-in game [:location] move [0 -1]) - \l (update-in game [:location] move [1 0]) - - \H (update-in game [:location] move [-5 0]) - \J (update-in game [:location] move [0 5]) - \K (update-in game [:location] move [0 -5]) - \L (update-in game [:location] move [5 0]) - - game)) - - (defmethod process-input :win [game input] - (if (= input :escape) - (assoc game :uis []) - (assoc game :uis [(->UI :start)]))) - - (defmethod process-input :lose [game input] - (if (= input :escape) - (assoc game :uis []) - (assoc game :uis [(->UI :start)]))) - - - (defn get-input [game screen] - (assoc game :input (s/get-key-blocking screen))) - -This isn't quite functional yet, because it needs the `->UI` factory function, -which is created by the `(defrecord UI [...])` that's still in `core.clj`. -I can't just `use` that in this file, because `core` is going to need to `use` -some functions from this, so I'd have circular imports. - -The solution is to move the `(defrecord UI [...])` into a separate file. -I chose to put it in `ui/core.clj`: - - :::clojure - (ns caves.ui.core) - - (defrecord UI [kind]) - -And now I can pull its creation function back into the `input` namespace: - - :::clojure - (ns caves.ui.input - (:use [caves.world :only [random-world smooth-world]] - [caves.ui.core :only [->UI]]) - (:require [lanterna.screen :as s])) - -Finally I can update the `ns` back in the original `core.clj` to pull in the -functions I need, and remove the ones I don't: - - :::clojure - (ns caves.core - (:use [caves.ui.core :only [->UI]] - [caves.ui.drawing :only [draw-game]] - [caves.ui.input :only [get-input process-input]]) - (:require [lanterna.screen :as s])) - -Whew! - -Results -------- - -That was a lot of shuffling around, but now I've got five separate files, each -pertaining to one specific thing, instead of one big pile of code. - -You can view the code [on GitHub][result-code] if you want to see the end -result. - -[result-code]: https://github.com/sjl/caves/tree/entry-03-4/src/caves - -Next post I swear I'll add the player so we'll have an actual game! - -{% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/07/caves-of-clojure-03-4.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2012/07/caves-of-clojure-03-4.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,346 @@ ++++ +title = "The Caves of Clojure: Part 3.4" +snip = "Refactoring." +date = 2012-07-11T12:02:00Z +draft = false + ++++ + +This post is part of an ongoing series. If you haven't already done so, you +should probably start at [the beginning][]. + +This entry (kind of) corresponds to [post three in Trystan's +tutorial][trystan-tut]. + +If you want to follow along, the code for the series is [on Bitbucket][bb] and +[on GitHub][gh]. Update to the `entry-03-4` tag to see the code as it stands +after this post. + +[the beginning]: /blog/2012/07/caves-of-clojure-01/ +[trystan-tut]: http://trystans.blogspot.com/2011/08/roguelike-tutorial-03-scrolling-through.html +[bb]: http://bitbucket.org/sjl/caves/ +[gh]: http://github.com/sjl/caves/ + +{{% toc %}} + +Summary +------- + +In the last post I said that the next post would be about Trystan's fourth +entry. I lied. I'm going to do a short entry about refactoring before I move +on, because I don't want to clutter up later ones with this stuff. + +Record Creation +--------------- + +Hacker News user bitsai [told me about][hn] a new syntax for creating records in +Clojure 1.3, so I updated all the `(new Foo)` calls to use that. + +One example is the `new-game` function. Before: + +[hn]: https://news.ycombinator.com/item?id=4220141 + +```clojure +(defn new-game [] + (assoc (new Game nil [(new UI :start)] nil) + :location [40 20])) +``` + +After: + +```clojure +(defn new-game [] + (assoc (->Game nil [(->UI :start)] nil) + :location [40 20])) +``` + +It's certainly not too impressive from a "characters saved" point of view. But +this little change matters more than it might appear at first glance. + +Imagine you define a record in Clojure: + +```clojure +(ns a) + +(defrecord Foo []) +``` + +And then in another namespace you want to use it: + +```clojure +(ns b + (:use a)) + +(new Foo) +``` + +This will explode, because `Foo` is actually a Java class, so you need to import +it: + +```clojure +(ns b + (:import a.Foo) + (:use a)) + +(new Foo) +``` + +This is one example of Clojure's Java underpinnings leaking through. + +In Clojure 1.3, `defrecord` will automatically generate a "factory function" +that creates the record, and you can `require` or `use` *that* like any other +function, so you don't need to screw around with a Java interop feature +(`import`) to use a pure Clojure feature. + +This is a good thing. It means that progress is being made toward patching the +places that Java leaks into Clojure. It gives me hope that some day I'll feel +okay recommending Clojure to people without Java experience. + +I updated all the `(new ...)` calls to use the new-style factory functions. +I won't paste them all here, but if you're following along you'll want to `grep +-R 'new ' .` and update the rest now. + +update-in +--------- + +Next is a tiny change that's just a bit cleaner. Alan Malloy [told me][] about +it. In the `process-input` function for the `:play` UI, the code to handle +smoothing the world looked like this: + +[told me]: https://twitter.com/alanmalloy/status/222748536595423232 + +```clojure +\s (assoc game :world (smooth-world (:world game))) +``` + +This can be done much more cleanly using `update-in`: + +```clojure +\s (update-in game [:world] smooth-world) +``` + +Nice. + +Namespaces +---------- + +I said in an earlier post that I tend to leave things in one file until I feel +like they need to be pulled out. Well, that time has come. + +First I pulled the UI drawing code into its own file: `ui/drawing.clj`. It +looks like this (nothing has changed, it's just in a file of its own now): + +```clojure +(ns caves.ui.drawing + (:require [lanterna.screen :as s])) + + +(def screen-size [80 24]) + +(defn clear-screen [screen] + (let [[cols rows] screen-size + blank (apply str (repeat cols \space))] + (doseq [row (range rows)] + (s/put-string screen 0 row blank)))) + + +(defmulti draw-ui + (fn [ui game screen] + (:kind ui))) + + +(defmethod draw-ui :start [ui game screen] + (s/put-string screen 0 0 "Welcome to the Caves of Clojure!") + (s/put-string screen 0 1 "Press any key to continue.") + (s/put-string screen 0 2 "") + (s/put-string screen 0 3 "Once in the game, you can use enter to win,") + (s/put-string screen 0 4 "and backspace to lose.")) + + +(defmethod draw-ui :win [ui game screen] + (s/put-string screen 0 0 "Congratulations, you win!") + (s/put-string screen 0 1 "Press escape to exit, anything else to restart.")) + + +(defmethod draw-ui :lose [ui game screen] + (s/put-string screen 0 0 "Sorry, better luck next time.") + (s/put-string screen 0 1 "Press escape to exit, anything else to restart.")) + + +(defn get-viewport-coords [game vcols vrows] + (let [location (:location game) + [center-x center-y] location + + tiles (:tiles (:world game)) + + map-rows (count tiles) + map-cols (count (first tiles)) + + start-x (- center-x (int (/ vcols 2))) + start-x (max 0 start-x) + + start-y (- center-y (int (/ vrows 2))) + start-y (max 0 start-y) + + end-x (+ start-x vcols) + end-x (min end-x map-cols) + + end-y (+ start-y vrows) + end-y (min end-y map-rows) + + start-x (- end-x vcols) + start-y (- end-y vrows)] + [start-x start-y end-x end-y])) + +(defn draw-crosshairs [screen vcols vrows] + (let [crosshair-x (int (/ vcols 2)) + crosshair-y (int (/ vrows 2))] + (s/put-string screen crosshair-x crosshair-y "X" {:fg :red}) + (s/move-cursor screen crosshair-x crosshair-y))) + +(defn draw-world [screen vrows vcols start-x start-y end-x end-y tiles] + (doseq [[vrow-idx mrow-idx] (map vector + (range 0 vrows) + (range start-y end-y)) + :let [row-tiles (subvec (tiles mrow-idx) start-x end-x)]] + (doseq [vcol-idx (range vcols) + :let [{:keys [glyph color]} (row-tiles vcol-idx)]] + (s/put-string screen vcol-idx vrow-idx glyph {:fg color})))) + +(defmethod draw-ui :play [ui game screen] + (let [world (:world game) + tiles (:tiles world) + [cols rows] screen-size + vcols cols + vrows (dec rows) + [start-x start-y end-x end-y] (get-viewport-coords game vcols vrows)] + (draw-world screen vrows vcols start-x start-y end-x end-y tiles) + (draw-crosshairs screen vcols vrows))) + + +(defn draw-game [game screen] + (clear-screen screen) + (doseq [ui (:uis game)] + (draw-ui ui game screen)) + (s/redraw screen)) +``` + +And now I need to `use` the `draw-game` function back in `core.clj`: + +```clojure +(ns caves.core + (:use [caves.world :only [random-world smooth-world]] + [caves.ui.drawing :only [draw-game]]) + (:require [lanterna.screen :as s])) +``` + +The fact that I moved eleven top-level symbols into a new namespace and only had +to bring one of them back into the original is a pretty clear sign that this +chunk of code was ready to be moved into its own file. + +I also did the same for the input processing code, moving it into +`ui/input.clj`: + +```clojure +(ns caves.ui.input + (:use [caves.world :only [random-world smooth-world]]) + (:require [lanterna.screen :as s])) + + +(defmulti process-input + (fn [game input] + (:kind (last (:uis game))))) + +(defmethod process-input :start [game input] + (-> game + (assoc :world (random-world)) + (assoc :uis [(->UI :play)]))) + + +(defn move [[x y] [dx dy]] + [(+ x dx) (+ y dy)]) + +(defmethod process-input :play [game input] + (case input + :enter (assoc game :uis [(->UI :win)]) + :backspace (assoc game :uis [(->UI :lose)]) + \q (assoc game :uis []) + + \s (update-in game [:world] smooth-world) + + \h (update-in game [:location] move [-1 0]) + \j (update-in game [:location] move [0 1]) + \k (update-in game [:location] move [0 -1]) + \l (update-in game [:location] move [1 0]) + + \H (update-in game [:location] move [-5 0]) + \J (update-in game [:location] move [0 5]) + \K (update-in game [:location] move [0 -5]) + \L (update-in game [:location] move [5 0]) + + game)) + +(defmethod process-input :win [game input] + (if (= input :escape) + (assoc game :uis []) + (assoc game :uis [(->UI :start)]))) + +(defmethod process-input :lose [game input] + (if (= input :escape) + (assoc game :uis []) + (assoc game :uis [(->UI :start)]))) + + +(defn get-input [game screen] + (assoc game :input (s/get-key-blocking screen))) +``` + +This isn't quite functional yet, because it needs the `->UI` factory function, +which is created by the `(defrecord UI [...])` that's still in `core.clj`. +I can't just `use` that in this file, because `core` is going to need to `use` +some functions from this, so I'd have circular imports. + +The solution is to move the `(defrecord UI [...])` into a separate file. +I chose to put it in `ui/core.clj`: + +```clojure +(ns caves.ui.core) + +(defrecord UI [kind]) +``` + +And now I can pull its creation function back into the `input` namespace: + +```clojure +(ns caves.ui.input + (:use [caves.world :only [random-world smooth-world]] + [caves.ui.core :only [->UI]]) + (:require [lanterna.screen :as s])) +``` + +Finally I can update the `ns` back in the original `core.clj` to pull in the +functions I need, and remove the ones I don't: + +```clojure +(ns caves.core + (:use [caves.ui.core :only [->UI]] + [caves.ui.drawing :only [draw-game]] + [caves.ui.input :only [get-input process-input]]) + (:require [lanterna.screen :as s])) +``` + +Whew! + +Results +------- + +That was a lot of shuffling around, but now I've got five separate files, each +pertaining to one specific thing, instead of one big pile of code. + +You can view the code [on GitHub][result-code] if you want to see the end +result. + +[result-code]: https://github.com/sjl/caves/tree/entry-03-4/src/caves + +Next post I swear I'll add the player so we'll have an actual game! + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/07/caves-of-clojure-04.html --- a/content/blog/2012/07/caves-of-clojure-04.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,623 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "The Caves of Clojure: Part 4" - snip: "A player!" - created: 2012-07-12 09:42:00 - %} - -{% block article %} - -This post is part of an ongoing series. If you haven't already done so, you -should probably start at [the beginning][]. - -This entry corresponds to [post four in Trystan's tutorial][trystan-tut]. - -If you want to follow along, the code for the series is [on Bitbucket][bb] and -[on GitHub][gh]. Update to the `entry-04` tag to see the code as it stands -after this post. - -[the beginning]: /blog/2012/07/caves-of-clojure-01/ -[trystan-tut]: http://trystans.blogspot.com/2011/08/roguelike-tutorial-04-player.html -[bb]: http://bitbucket.org/sjl/caves/ -[gh]: http://github.com/sjl/caves/ - -[TOC] - -Summary -------- - -In Trystan's fourth post he adds three main things: - -* A player -* Player movement -* Digging - -I'm going to add all three of those, but I'm going to do things very differently -than he did. - -My goal is to play around with some Clojurey concepts and see how far I can -stretch them. I have a feeling that it's going to let me do some very cool -things in the future. - -Refactoring ------------ - -Before I started I wanted to clean up the `world` namespace a bit. I'm not -going to go in depth -- I'll just post the code and you can read over it or skip -it if you trust me. - -First I created `coords.clj`: - - :::clojure - (ns caves.coords) - - - (defn offset-coords - "Offset the starting coordinate by the given amount, returning the result coordinate." - [[x y] [dx dy]] - [(+ x dx) (+ y dy)]) - - (defn dir-to-offset - "Convert a direction to the offset for moving 1 in that direction." - [dir] - (case dir - :w [-1 0] - :e [1 0] - :n [0 -1] - :s [0 1] - :nw [-1 -1] - :ne [1 -1] - :sw [-1 1] - :se [1 1])) - - (defn destination-coords - "Take an origin's coords and a direction and return the destination's coords." - [origin dir] - (offset-coords origin (dir-to-offset dir))) - -Then I cleaned up `world.clj`: - - :::clojure - (ns caves.world) - - - ; Constants ------------------------------------------------------------------- - (def world-size [160 50]) - - ; Data structures ------------------------------------------------------------- - (defrecord World [tiles]) - (defrecord Tile [kind glyph color]) - - (def tiles - {:floor (->Tile :floor "." :white) - :wall (->Tile :wall "#" :white) - :bound (->Tile :bound "X" :black)}) - - - ; Convenience functions ------------------------------------------------------- - (defn get-tile-from-tiles [tiles [x y]] - (get-in tiles [y x] (:bound tiles))) - - (defn random-coordinate [] - (let [[cols rows] world-size] - [(rand-int cols) (rand-int rows)])) - - - ; World generation ------------------------------------------------------------ - (defn random-tiles [] - (let [[cols rows] world-size] - (letfn [(random-tile [] - (tiles (rand-nth [:floor :wall]))) - (random-row [] - (vec (repeatedly cols random-tile)))] - (vec (repeatedly rows random-row))))) - - - (defn get-smoothed-tile [block] - (let [tile-counts (frequencies (map :kind block)) - floor-threshold 5 - floor-count (get tile-counts :floor 0) - result (if (>= floor-count floor-threshold) - :floor - :wall)] - (tiles result))) - - (defn block-coords [x y] - (for [dx [-1 0 1] - dy [-1 0 1]] - [(+ x dx) (+ y dy)])) - - (defn get-block [tiles x y] - (map (partial get-tile-from-tiles tiles) - (block-coords x y))) - - (defn get-smoothed-row [tiles y] - (mapv (fn [x] - (get-smoothed-tile (get-block tiles x y))) - (range (count (first tiles))))) - - (defn get-smoothed-tiles [tiles] - (mapv (fn [y] - (get-smoothed-row tiles y)) - (range (count tiles)))) - - (defn smooth-world [{:keys [tiles] :as world}] - (assoc world :tiles (get-smoothed-tiles tiles))) - - - (defn random-world [] - (let [world (->World (random-tiles)) - world (nth (iterate smooth-world world) 3)] - world)) - -The changes were mostly centered around removing debugging functions and making -all the world functions take an `[x y]` coordinate vector in place of two -separate `x` and `y` arguments. - -Read through the code if you're curious, it's only about 100 total lines. - -Entities --------- - -Now it's time to add a player. Rather than take the approach Trystan used of -creating a `Creature` class and `Player` subclass, I went with something a bit -different. - -In Minecraft's network protocol, when you get a list of "things in the world" -it's not just creatures -- the list includes both creatures and items. It uses -the word "entity" to refer to them. I'm sure it's not the first game to do -that, but it's the first time I've run across it because I'm not hugely into -game programming. - -That got me thinking: are items and creatures really so different that I need to -represent them as two completely separate ideas? - -Both have a location in the world. Both have a "glyph" that I'll be using to -display them to the player. Both will have some kind of "id" so I can use them -in mappings efficiently. - -On the other hand, there are definitely some differences. Creatures move -around, eat things, attack things, can be attacked (and killed), have an AI to -decide what to do, and so on. - -Items can be picked up and dropped, can contain other items, can be eaten or -quaffed, can rot over time (e.g.: corpses), can be used as weapons or armor, et -cetera. - -But wait a second: are things really so clear? The bag of tricks in Nethack can -attack the player. Cockatrice corpses can be wielded and used as -petrification-inducing clubs. In Dwarf Fortress discarded pieces from -slaughtered animals can come alive if a necromancer sieges your fortress. - -I can think of a lot of cool things I could do when I eliminate the distinction -between items and creatures. - -Maybe there's a "pixie" creature that wanders around and does normal creaturey -things, but if you attack it while wielding a butterfly net you "catch" it and -it gets picked up and put in your inventory like an item. - -Once you've got one you could "apply" it (maybe that means "setting it free") to -heal yourself, or quaff it to restore mana (mmm, delicious pixie blood). - -Oh, and it still has an AI so maybe every 100 turns it has a chance of escaping -from your inventory. Unless you put it in a jar. - -I can think of tons of interesting things to do with a unified "entity" system. -A bag that eat things, where you need to remember to "feed" it normal food or -it'll start digesting the other items! Giant venus fly traps that eat unwary -pixies! Potions that evaporate over time if you don't use them! - -The possibilities are really exciting. But how can I actually *code* all this -crazy stuff without special-casing *everything*? - -Protocols ---------- - -After thinking about this problem for a while, I came up with a solution that -I think has some real promise. - -Individual types of entity ("pixie", "player", "goblin", "steel helmet") will be -defined with simple `(defrecord)`s. Each should have an `:id`, `:glyph`, and -`:location`, but beyond that the rest of their state is flexible. - -I'm going to create an `Entity` protocol that such records will implement. That -protocol will have a single `tick` function that they need to define. This will -be called once per game "tick" and will be how the various types of entity -decide what to do over time. They may define a `tick` that does nothing if they -don't change over time. - -On its own an entity record can't do anything except exist, be displayed on -the map, and update itself every tick. To actually *do* something during -a tick (or have things done to them) they'll implement what I'm calling -"aspects". - -An "aspect" is a protocol that defines a group of related functions, probably -all having to do with a simple gameplay mechanic. Here are a few rough examples -from the top of my head: - - :::clojure - (defprotocol Edible - (can-be-eaten? [this eater world]) - (nutrition-value [this world]) - (eat [this eater world])) - - (defprotocol Eater - (can-eat? [this food world]) - (eat [this food world])) - - (defprotocol Item - (can-be-contained-in? [this container world]) - (insert-into [this container world]) - (remove-from [this container world])) - - (defprotocol Container - (get-contained [this world]) - (can-contain? [this item world]) - (insert [this item world]) - (remove [this item world])) - -As you can see, many aspects will be paired up. Some entities can have things -done to them by other entities, which will actually do those things. Both will -have the opportunity to override the default method implementations to customize -the behavior. - -Anyway, I think this way of adding in functionality (basically mixin-style, but -decoupled from the entity class declaration and without namespace clashes) -could be very cool. I'm going to give it a shot and see how it works. - -The Player ----------- - -Let's start with the first and most important entity: the player. This game -isn't going to be much fun without one of those. - -First I added a new file: `entities/core.clj`. It'll contain the basic `Entity` -definition: - - :::clojure - (ns caves.entities.core) - - - (defprotocol Entity - (tick [this world] - "Update the world to handle the passing of a tick for this entity.")) - -Simple enough. `tick`ing an entity will return a new immutable world that -accounts for whatever the entity decides to do during that tick. - -Now to add a player! - - :::clojure - (ns caves.entities.player - (:use [caves.entities.core :only [Entity]])) - - - (defrecord Player [id glyph location]) - - (extend-type Player Entity - (tick [this world] - world)) - -Right now the player doesn't do anything during a tick -- the world will remain -unchanged. - -We'll need to actually place the player somewhere in the world to start, so I'll -make a helper function like Trystan's to find an empty spot for them in -`world.clj`: - - :::clojure - (defn find-empty-tile [world] - (loop [coord (random-coordinate)] - (if (#{:floor} (get-tile-kind world coord)) - coord - (recur (random-coordinate))))) - -Basically I just try a bunch of random coordinates until I find one that's -a `:floor` tile. Maybe not the most efficient way to do things, but it's fine -for now. - -Back in `player.clj` I'll need a way to make a new player when we start a new -game: - - :::clojure - (defn make-player [world] - (->Player :player "@" (find-empty-tile world))) - -For now I'll use the special ID `:player` for the entity ID. Since this is -going to be a single player game, with no chance of ever being multiplayer, it's -okay to special case things for the player a bit. - -Now to actually add the new player into the main `game` object. Remember that -the `:start` screen is the one that makes fresh games, so I updated that: - - :::clojure - (defn reset-game [game] - (let [fresh-world (random-world)] - (-> game - (assoc :world fresh-world) - (assoc-in [:world :player] (make-player fresh-world)) - (assoc :uis [(->UI :play)])))) - - (defmethod process-input :start [game input] - (reset-game game)) - -I pulled out the guts of the `process-input` function into a helper, which: - -* Creates a fresh, random world. -* Replaces the game's world with the new one. -* Creates a fresh player at some empty location in that world. -* Attaches the player to the world. -* Replaces the UI stack of the game with the main `:play` UI. - -I could have made a completely new `game` object instead of just overwriting -some fields here, but this way if I decide to store configuration options on the -`game` later they won't be lost when restarting. - -Displaying the Player ---------------------- - -Now that I've got a player it's time to display them on the map as the -traditional `@`. I opened up `input.clj` and replaced the crosshair-drawing -code from the last two posts with code to draw the player: - - :::clojure - (defn draw-player [screen start-x start-y player] - (let [[player-x player-y] (:location player) - x (- player-x start-x) - y (- player-y start-y)] - (s/put-string screen x y (:glyph player) {:fg :white}) - (s/move-cursor screen x y))) - -If the screen's `start-x` (i.e.: its left edge) is at 10, and the player is at -24, then I need to draw the `@` at screen coordinate 14. Same goes for the -y coordinates. - -Now to tweak the main `draw-ui` function to account for this change: - - :::clojure - (defmethod draw-ui :play [ui game screen] - (let [world (:world game) - {:keys [tiles player]} world - [cols rows] screen-size - vcols cols - vrows (dec rows) - [start-x start-y end-x end-y] (get-viewport-coords game (:location player) vcols vrows)] - (draw-world screen vrows vcols start-x start-y end-x end-y tiles) - (draw-player screen start-x start-y player))) - -Instead of a `center-x` and `center-y` that are based on an arbitrary value -in the `game` object, I'm now basing things off of the player's coordinates. -Otherwise not much has changed here. - -One more thing I decided to do is the start using that line at the bottom of the -screen that I reserved for stats. I added a simple function to draw it: - - :::clojure - (defn draw-hud [screen game start-x start-y] - (let [hud-row (dec (second screen-size)) - [x y] (get-in game [:world :player :location]) - info (str "loc: [" x "-" y "]") - info (str info " start: [" start-x "-" start-y "]")] - (s/put-string screen 0 hud-row info))) - -And add a call for that to the `draw-ui` function. I'm sure you can figure that -out yourself. - -Now the last line of the screen will look like: - - :::text - loc: [30-53] start: [10-34] - -I can now see the coordinates of the player and the top left corner of the -screen at all times. This was really handy when debugging display/movement -problems later. - -Movement --------- - -Now that I'm basing the viewport on the player's location, I need a way for -players to move around. I could just tweak the current code, but lots of things -are going to need to move so this sounds like a great place for my first -"aspect". - -I created the `entities/aspects/mobile.clj` file and added the protocol -representing the aspect: - - :::clojure - (ns caves.entities.aspects.mobile) - - - (defprotocol Mobile - (move [this world dest] - "Move this entity to a new location.") - (can-move? [this world dest] - "Return whether the entity can move to the new location.")) - -Right now I'm just defining some simple functions. Mobile entities must be able -to check if they can move into a coordinate, as well as actually move themselves -into it. - -Why allow entities to check for movement and move themselves instead of having -a single movement handling chunk of code for all entities? - -Well this means we can customize how movement works on a per-entity basis. -Maybe we'll have a minotaur that can move into other entities' spaces, -displacing them. Or a stone elemental that can walk through wall tiles. - -Next I made the Player entity implement Mobile back in `entities/player.clj`: - - :::clojure - (ns caves.entities.player - (:use [caves.entities.core :only [Entity]] - [caves.entities.aspects.mobile :only [Mobile move can-move?]] - [caves.coords :only [destination-coords]] - [caves.world :only [find-empty-tile get-tile-kind]])) - - - (defrecord Player [id glyph location]) - - (defn check-tile - "Check that the tile at the destination passes the given predicate." - [world dest pred] - (pred (get-tile-kind world dest))) - - - (extend-type Player Entity - (tick [this world] - world)) - - (extend-type Player Mobile - (move [this world dest] - {:pre [(can-move? this world dest)]} - (assoc-in world [:player :location] dest)) - (can-move? [this world dest] - (check-tile world dest #{:floor}))) - - - (defn make-player [world] - (->Player :player "@" (find-empty-tile world))) - - (defn move-player [world dir] - (let [player (:player world) - target (destination-coords (:location player) dir)] - (cond - (can-move? player world target) (move player world target) - :else world))) - -Notice how simple (and concise) this was to add. I defined `can-move?` to -simply make sure that the destination is a floor tile. - -`move` itself uses a Clojure function precondition to sanity-check that the -entity isn't trying to cheat and move somewhere illegal. If everything's okay, -I simply update the player's location. - -`move-player` is an ugly helper function that most entities won't need. Players -are special because we're going to want to make certain keystrokes do multiple -things, as we'll see shortly. For now don't worry too much about that one. - -Before going on make sure you understand how movement is actually going to -happen, from the point where `(move-player game :s)` is called and down. - -The last thing to do to actually make the player movable is handling the actual -keystrokes from the user, so I did that next over in `ui/input.clj`: - - :::clojure - (defmethod process-input :play [game input] - (case input - :enter (assoc game :uis [(->UI :win)]) - :backspace (assoc game :uis [(->UI :lose)]) - \q (assoc game :uis []) - - \h (update-in game [:world] move-player :w) - \j (update-in game [:world] move-player :s) - \k (update-in game [:world] move-player :n) - \l (update-in game [:world] move-player :e) - \y (update-in game [:world] move-player :nw) - \u (update-in game [:world] move-player :ne) - \b (update-in game [:world] move-player :sw) - \n (update-in game [:world] move-player :se) - - game)) - -Each of the traditional roguelike movement keys will now move the player around -the world. Because the screen drawing is already updated to be based on the -player's location, movement is pretty much complete! - -I added the `yubn` diagonal movement keys because as I was trying out movement -myself it felt like I needed them. - -This is something that I've noticed while watching Notch's Ludum Dare recordings -(Google for them if you want to see them). He plays the game he's making for -longer periods than you might think. He doesn't just make a feature and make -sure it works, he makes a feature and then plays the game normally for a few -minutes to make sure it fits into the game right (and is fun)! - -Those `update-in` statements are a bit ugly, but not ugly enough for me to want -to do something clever to remove them. They can stay for now. - -Digging -------- - -As Trystan mentioned in his post, we're not doing anything special to make sure -the caves we generate are connected. The player may very well start in a tiny -cave. - -To make this less of a problem, he added the ability for the player to dig -through walls. - -Digging sounds like a great candidate for another aspect, so I added -`entities/aspects/digger.clj`: - - :::clojure - (ns caves.entities.aspects.digger) - - - (defprotocol Digger - (dig [this world target] - "Dig a location.") - (can-dig? [this world target] - "Return whether the entity can dig the new location.")) - -Nothing fancy here. Then I made the Player entity implement it: - - :::clojure - (extend-type Player Digger - (dig [this world dest] - {:pre [(can-dig? this world dest)]} - (set-tile-floor world dest)) - (can-dig? [this world dest] - (check-tile world dest #{:wall}))) - -This looks very similar to the Mobile implementation, except instead of changing -the player's location I change the map tile from a wall to a floor. - -Finally, I update the `move-player` function (which is called when we receive -a keystroke): - - :::clojure - (defn move-player [world dir] - (let [player (:player world) - target (destination-coords (:location player) dir)] - (cond - (can-move? player world target) (move player world target) - (can-dig? player world target) (dig player world target) - :else world))) - -Now if the space the user is telling the player to enter is open, the player -will move there, otherwise if it's diggable the player will dig it, otherwise -nothing will happen. - -This means that moving into a space that is currently a wall will take two -keypresses: the first digs out the wall, the second moves into the newly open -space. - -I like how this feels. It takes longer to travel through rock, which makes -sense. If you prefer to dig and move all at once you could dig and move in the -same action. It's up to you. - -Results -------- - -Finally, after seven entries I've got a hero in the game! It's taken a while, -but I've laid the groundwork for what I think is some really cool stuff down the -line. - -You can view the code [on GitHub][result-code] if you want to see the end -result. From now on I'm going to start moving a bit faster, not always showing -the namespace declarations and such. If you want the full code for each post -look at the GitHub repository. - -[result-code]: https://github.com/sjl/caves/tree/entry-04/src/caves - -And the obligatory screenshots of our intrepid hero: - -![Screenshot](/media/images{{ parent_url }}/caves-04-01.png) - -![Screenshot](/media/images{{ parent_url }}/caves-04-02.png) - -![Screenshot](/media/images{{ parent_url }}/caves-04-03.png) - -Next time I'll be adding some monsters for the hero to slay. - -{% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/07/caves-of-clojure-04.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2012/07/caves-of-clojure-04.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,638 @@ ++++ +title = "The Caves of Clojure: Part 4" +snip = "A player!" +date = 2012-07-12T09:42:00Z +draft = false + ++++ + +This post is part of an ongoing series. If you haven't already done so, you +should probably start at [the beginning][]. + +This entry corresponds to [post four in Trystan's tutorial][trystan-tut]. + +If you want to follow along, the code for the series is [on Bitbucket][bb] and +[on GitHub][gh]. Update to the `entry-04` tag to see the code as it stands +after this post. + +[the beginning]: /blog/2012/07/caves-of-clojure-01/ +[trystan-tut]: http://trystans.blogspot.com/2011/08/roguelike-tutorial-04-player.html +[bb]: http://bitbucket.org/sjl/caves/ +[gh]: http://github.com/sjl/caves/ + +{{% toc %}} + +Summary +------- + +In Trystan's fourth post he adds three main things: + +* A player +* Player movement +* Digging + +I'm going to add all three of those, but I'm going to do things very differently +than he did. + +My goal is to play around with some Clojurey concepts and see how far I can +stretch them. I have a feeling that it's going to let me do some very cool +things in the future. + +Refactoring +----------- + +Before I started I wanted to clean up the `world` namespace a bit. I'm not +going to go in depth — I'll just post the code and you can read over it or skip +it if you trust me. + +First I created `coords.clj`: + +```clojure +(ns caves.coords) + + +(defn offset-coords + "Offset the starting coordinate by the given amount, returning the result coordinate." + [[x y] [dx dy]] + [(+ x dx) (+ y dy)]) + +(defn dir-to-offset + "Convert a direction to the offset for moving 1 in that direction." + [dir] + (case dir + :w [-1 0] + :e [1 0] + :n [0 -1] + :s [0 1] + :nw [-1 -1] + :ne [1 -1] + :sw [-1 1] + :se [1 1])) + +(defn destination-coords + "Take an origin's coords and a direction and return the destination's coords." + [origin dir] + (offset-coords origin (dir-to-offset dir))) +``` + +Then I cleaned up `world.clj`: + +```clojure +(ns caves.world) + + +; Constants ------------------------------------------------------------------- +(def world-size [160 50]) + +; Data structures ------------------------------------------------------------- +(defrecord World [tiles]) +(defrecord Tile [kind glyph color]) + +(def tiles + {:floor (->Tile :floor "." :white) + :wall (->Tile :wall "#" :white) + :bound (->Tile :bound "X" :black)}) + + +; Convenience functions ------------------------------------------------------- +(defn get-tile-from-tiles [tiles [x y]] + (get-in tiles [y x] (:bound tiles))) + +(defn random-coordinate [] + (let [[cols rows] world-size] + [(rand-int cols) (rand-int rows)])) + + +; World generation ------------------------------------------------------------ +(defn random-tiles [] + (let [[cols rows] world-size] + (letfn [(random-tile [] + (tiles (rand-nth [:floor :wall]))) + (random-row [] + (vec (repeatedly cols random-tile)))] + (vec (repeatedly rows random-row))))) + + +(defn get-smoothed-tile [block] + (let [tile-counts (frequencies (map :kind block)) + floor-threshold 5 + floor-count (get tile-counts :floor 0) + result (if (>= floor-count floor-threshold) + :floor + :wall)] + (tiles result))) + +(defn block-coords [x y] + (for [dx [-1 0 1] + dy [-1 0 1]] + [(+ x dx) (+ y dy)])) + +(defn get-block [tiles x y] + (map (partial get-tile-from-tiles tiles) + (block-coords x y))) + +(defn get-smoothed-row [tiles y] + (mapv (fn [x] + (get-smoothed-tile (get-block tiles x y))) + (range (count (first tiles))))) + +(defn get-smoothed-tiles [tiles] + (mapv (fn [y] + (get-smoothed-row tiles y)) + (range (count tiles)))) + +(defn smooth-world [{:keys [tiles] :as world}] + (assoc world :tiles (get-smoothed-tiles tiles))) + + +(defn random-world [] + (let [world (->World (random-tiles)) + world (nth (iterate smooth-world world) 3)] + world)) +``` + +The changes were mostly centered around removing debugging functions and making +all the world functions take an `[x y]` coordinate vector in place of two +separate `x` and `y` arguments. + +Read through the code if you're curious, it's only about 100 total lines. + +Entities +-------- + +Now it's time to add a player. Rather than take the approach Trystan used of +creating a `Creature` class and `Player` subclass, I went with something a bit +different. + +In Minecraft's network protocol, when you get a list of "things in the world" +it's not just creatures — the list includes both creatures and items. It uses +the word "entity" to refer to them. I'm sure it's not the first game to do +that, but it's the first time I've run across it because I'm not hugely into +game programming. + +That got me thinking: are items and creatures really so different that I need to +represent them as two completely separate ideas? + +Both have a location in the world. Both have a "glyph" that I'll be using to +display them to the player. Both will have some kind of "id" so I can use them +in mappings efficiently. + +On the other hand, there are definitely some differences. Creatures move +around, eat things, attack things, can be attacked (and killed), have an AI to +decide what to do, and so on. + +Items can be picked up and dropped, can contain other items, can be eaten or +quaffed, can rot over time (e.g.: corpses), can be used as weapons or armor, et +cetera. + +But wait a second: are things really so clear? The bag of tricks in Nethack can +attack the player. Cockatrice corpses can be wielded and used as +petrification-inducing clubs. In Dwarf Fortress discarded pieces from +slaughtered animals can come alive if a necromancer sieges your fortress. + +I can think of a lot of cool things I could do when I eliminate the distinction +between items and creatures. + +Maybe there's a "pixie" creature that wanders around and does normal creaturey +things, but if you attack it while wielding a butterfly net you "catch" it and +it gets picked up and put in your inventory like an item. + +Once you've got one you could "apply" it (maybe that means "setting it free") to +heal yourself, or quaff it to restore mana (mmm, delicious pixie blood). + +Oh, and it still has an AI so maybe every 100 turns it has a chance of escaping +from your inventory. Unless you put it in a jar. + +I can think of tons of interesting things to do with a unified "entity" system. +A bag that eat things, where you need to remember to "feed" it normal food or +it'll start digesting the other items! Giant venus fly traps that eat unwary +pixies! Potions that evaporate over time if you don't use them! + +The possibilities are really exciting. But how can I actually *code* all this +crazy stuff without special-casing *everything*? + +Protocols +--------- + +After thinking about this problem for a while, I came up with a solution that +I think has some real promise. + +Individual types of entity ("pixie", "player", "goblin", "steel helmet") will be +defined with simple `(defrecord)`s. Each should have an `:id`, `:glyph`, and +`:location`, but beyond that the rest of their state is flexible. + +I'm going to create an `Entity` protocol that such records will implement. That +protocol will have a single `tick` function that they need to define. This will +be called once per game "tick" and will be how the various types of entity +decide what to do over time. They may define a `tick` that does nothing if they +don't change over time. + +On its own an entity record can't do anything except exist, be displayed on +the map, and update itself every tick. To actually *do* something during +a tick (or have things done to them) they'll implement what I'm calling +"aspects". + +An "aspect" is a protocol that defines a group of related functions, probably +all having to do with a simple gameplay mechanic. Here are a few rough examples +from the top of my head: + +```clojure +(defprotocol Edible + (can-be-eaten? [this eater world]) + (nutrition-value [this world]) + (eat [this eater world])) + +(defprotocol Eater + (can-eat? [this food world]) + (eat [this food world])) + +(defprotocol Item + (can-be-contained-in? [this container world]) + (insert-into [this container world]) + (remove-from [this container world])) + +(defprotocol Container + (get-contained [this world]) + (can-contain? [this item world]) + (insert [this item world]) + (remove [this item world])) +``` + +As you can see, many aspects will be paired up. Some entities can have things +done to them by other entities, which will actually do those things. Both will +have the opportunity to override the default method implementations to customize +the behavior. + +Anyway, I think this way of adding in functionality (basically mixin-style, but +decoupled from the entity class declaration and without namespace clashes) +could be very cool. I'm going to give it a shot and see how it works. + +The Player +---------- + +Let's start with the first and most important entity: the player. This game +isn't going to be much fun without one of those. + +First I added a new file: `entities/core.clj`. It'll contain the basic `Entity` +definition: + +```clojure +(ns caves.entities.core) + + +(defprotocol Entity + (tick [this world] + "Update the world to handle the passing of a tick for this entity.")) +``` + +Simple enough. `tick`ing an entity will return a new immutable world that +accounts for whatever the entity decides to do during that tick. + +Now to add a player! + +```clojure +(ns caves.entities.player + (:use [caves.entities.core :only [Entity]])) + + +(defrecord Player [id glyph location]) + +(extend-type Player Entity + (tick [this world] + world)) +``` + +Right now the player doesn't do anything during a tick — the world will remain +unchanged. + +We'll need to actually place the player somewhere in the world to start, so I'll +make a helper function like Trystan's to find an empty spot for them in +`world.clj`: + +```clojure +(defn find-empty-tile [world] + (loop [coord (random-coordinate)] + (if (#{:floor} (get-tile-kind world coord)) + coord + (recur (random-coordinate))))) +``` + +Basically I just try a bunch of random coordinates until I find one that's +a `:floor` tile. Maybe not the most efficient way to do things, but it's fine +for now. + +Back in `player.clj` I'll need a way to make a new player when we start a new +game: + +```clojure +(defn make-player [world] + (->Player :player "@" (find-empty-tile world))) +``` + +For now I'll use the special ID `:player` for the entity ID. Since this is +going to be a single player game, with no chance of ever being multiplayer, it's +okay to special case things for the player a bit. + +Now to actually add the new player into the main `game` object. Remember that +the `:start` screen is the one that makes fresh games, so I updated that: + +```clojure +(defn reset-game [game] + (let [fresh-world (random-world)] + (-> game + (assoc :world fresh-world) + (assoc-in [:world :player] (make-player fresh-world)) + (assoc :uis [(->UI :play)])))) + +(defmethod process-input :start [game input] + (reset-game game)) +``` + +I pulled out the guts of the `process-input` function into a helper, which: + +* Creates a fresh, random world. +* Replaces the game's world with the new one. +* Creates a fresh player at some empty location in that world. +* Attaches the player to the world. +* Replaces the UI stack of the game with the main `:play` UI. + +I could have made a completely new `game` object instead of just overwriting +some fields here, but this way if I decide to store configuration options on the +`game` later they won't be lost when restarting. + +Displaying the Player +--------------------- + +Now that I've got a player it's time to display them on the map as the +traditional `@`. I opened up `input.clj` and replaced the crosshair-drawing +code from the last two posts with code to draw the player: + +```clojure +(defn draw-player [screen start-x start-y player] + (let [[player-x player-y] (:location player) + x (- player-x start-x) + y (- player-y start-y)] + (s/put-string screen x y (:glyph player) {:fg :white}) + (s/move-cursor screen x y))) +``` + +If the screen's `start-x` (i.e.: its left edge) is at 10, and the player is at +24, then I need to draw the `@` at screen coordinate 14. Same goes for the +y coordinates. + +Now to tweak the main `draw-ui` function to account for this change: + +```clojure +(defmethod draw-ui :play [ui game screen] + (let [world (:world game) + {:keys [tiles player]} world + [cols rows] screen-size + vcols cols + vrows (dec rows) + [start-x start-y end-x end-y] (get-viewport-coords game (:location player) vcols vrows)] + (draw-world screen vrows vcols start-x start-y end-x end-y tiles) + (draw-player screen start-x start-y player))) +``` + +Instead of a `center-x` and `center-y` that are based on an arbitrary value +in the `game` object, I'm now basing things off of the player's coordinates. +Otherwise not much has changed here. + +One more thing I decided to do is the start using that line at the bottom of the +screen that I reserved for stats. I added a simple function to draw it: + +```clojure +(defn draw-hud [screen game start-x start-y] + (let [hud-row (dec (second screen-size)) + [x y] (get-in game [:world :player :location]) + info (str "loc: [" x "-" y "]") + info (str info " start: [" start-x "-" start-y "]")] + (s/put-string screen 0 hud-row info))) +``` + +And add a call for that to the `draw-ui` function. I'm sure you can figure that +out yourself. + +Now the last line of the screen will look like: + +```text +loc: [30-53] start: [10-34] +``` + +I can now see the coordinates of the player and the top left corner of the +screen at all times. This was really handy when debugging display/movement +problems later. + +Movement +-------- + +Now that I'm basing the viewport on the player's location, I need a way for +players to move around. I could just tweak the current code, but lots of things +are going to need to move so this sounds like a great place for my first +"aspect". + +I created the `entities/aspects/mobile.clj` file and added the protocol +representing the aspect: + +```clojure +(ns caves.entities.aspects.mobile) + + +(defprotocol Mobile + (move [this world dest] + "Move this entity to a new location.") + (can-move? [this world dest] + "Return whether the entity can move to the new location.")) +``` + +Right now I'm just defining some simple functions. Mobile entities must be able +to check if they can move into a coordinate, as well as actually move themselves +into it. + +Why allow entities to check for movement and move themselves instead of having +a single movement handling chunk of code for all entities? + +Well this means we can customize how movement works on a per-entity basis. +Maybe we'll have a minotaur that can move into other entities' spaces, +displacing them. Or a stone elemental that can walk through wall tiles. + +Next I made the Player entity implement Mobile back in `entities/player.clj`: + +```clojure +(ns caves.entities.player + (:use [caves.entities.core :only [Entity]] + [caves.entities.aspects.mobile :only [Mobile move can-move?]] + [caves.coords :only [destination-coords]] + [caves.world :only [find-empty-tile get-tile-kind]])) + + +(defrecord Player [id glyph location]) + +(defn check-tile + "Check that the tile at the destination passes the given predicate." + [world dest pred] + (pred (get-tile-kind world dest))) + + +(extend-type Player Entity + (tick [this world] + world)) + +(extend-type Player Mobile + (move [this world dest] + {:pre [(can-move? this world dest)]} + (assoc-in world [:player :location] dest)) + (can-move? [this world dest] + (check-tile world dest #{:floor}))) + + +(defn make-player [world] + (->Player :player "@" (find-empty-tile world))) + +(defn move-player [world dir] + (let [player (:player world) + target (destination-coords (:location player) dir)] + (cond + (can-move? player world target) (move player world target) + :else world))) +``` + +Notice how simple (and concise) this was to add. I defined `can-move?` to +simply make sure that the destination is a floor tile. + +`move` itself uses a Clojure function precondition to sanity-check that the +entity isn't trying to cheat and move somewhere illegal. If everything's okay, +I simply update the player's location. + +`move-player` is an ugly helper function that most entities won't need. Players +are special because we're going to want to make certain keystrokes do multiple +things, as we'll see shortly. For now don't worry too much about that one. + +Before going on make sure you understand how movement is actually going to +happen, from the point where `(move-player game :s)` is called and down. + +The last thing to do to actually make the player movable is handling the actual +keystrokes from the user, so I did that next over in `ui/input.clj`: + +```clojure +(defmethod process-input :play [game input] + (case input + :enter (assoc game :uis [(->UI :win)]) + :backspace (assoc game :uis [(->UI :lose)]) + \q (assoc game :uis []) + + \h (update-in game [:world] move-player :w) + \j (update-in game [:world] move-player :s) + \k (update-in game [:world] move-player :n) + \l (update-in game [:world] move-player :e) + \y (update-in game [:world] move-player :nw) + \u (update-in game [:world] move-player :ne) + \b (update-in game [:world] move-player :sw) + \n (update-in game [:world] move-player :se) + + game)) +``` + +Each of the traditional roguelike movement keys will now move the player around +the world. Because the screen drawing is already updated to be based on the +player's location, movement is pretty much complete! + +I added the `yubn` diagonal movement keys because as I was trying out movement +myself it felt like I needed them. + +This is something that I've noticed while watching Notch's Ludum Dare recordings +(Google for them if you want to see them). He plays the game he's making for +longer periods than you might think. He doesn't just make a feature and make +sure it works, he makes a feature and then plays the game normally for a few +minutes to make sure it fits into the game right (and is fun)! + +Those `update-in` statements are a bit ugly, but not ugly enough for me to want +to do something clever to remove them. They can stay for now. + +Digging +------- + +As Trystan mentioned in his post, we're not doing anything special to make sure +the caves we generate are connected. The player may very well start in a tiny +cave. + +To make this less of a problem, he added the ability for the player to dig +through walls. + +Digging sounds like a great candidate for another aspect, so I added +`entities/aspects/digger.clj`: + +```clojure +(ns caves.entities.aspects.digger) + + +(defprotocol Digger + (dig [this world target] + "Dig a location.") + (can-dig? [this world target] + "Return whether the entity can dig the new location.")) +``` + +Nothing fancy here. Then I made the Player entity implement it: + +```clojure +(extend-type Player Digger + (dig [this world dest] + {:pre [(can-dig? this world dest)]} + (set-tile-floor world dest)) + (can-dig? [this world dest] + (check-tile world dest #{:wall}))) +``` + +This looks very similar to the Mobile implementation, except instead of changing +the player's location I change the map tile from a wall to a floor. + +Finally, I update the `move-player` function (which is called when we receive +a keystroke): + +```clojure +(defn move-player [world dir] + (let [player (:player world) + target (destination-coords (:location player) dir)] + (cond + (can-move? player world target) (move player world target) + (can-dig? player world target) (dig player world target) + :else world))) +``` + +Now if the space the user is telling the player to enter is open, the player +will move there, otherwise if it's diggable the player will dig it, otherwise +nothing will happen. + +This means that moving into a space that is currently a wall will take two +keypresses: the first digs out the wall, the second moves into the newly open +space. + +I like how this feels. It takes longer to travel through rock, which makes +sense. If you prefer to dig and move all at once you could dig and move in the +same action. It's up to you. + +Results +------- + +Finally, after seven entries I've got a hero in the game! It's taken a while, +but I've laid the groundwork for what I think is some really cool stuff down the +line. + +You can view the code [on GitHub][result-code] if you want to see the end +result. From now on I'm going to start moving a bit faster, not always showing +the namespace declarations and such. If you want the full code for each post +look at the GitHub repository. + +[result-code]: https://github.com/sjl/caves/tree/entry-04/src/caves + +And the obligatory screenshots of our intrepid hero: + +![Screenshot](/media/images/blog/2012/07/caves-04-01.png) + +![Screenshot](/media/images/blog/2012/07/caves-04-02.png) + +![Screenshot](/media/images/blog/2012/07/caves-04-03.png) + +Next time I'll be adding some monsters for the hero to slay. + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/07/caves-of-clojure-05.html --- a/content/blog/2012/07/caves-of-clojure-05.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,480 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "The Caves of Clojure: Part 5" - snip: "Fungus and more." - created: 2012-07-13 10:55:00 - %} - -{% block article %} - -This post is part of an ongoing series. If you haven't already done so, you -should probably start at [the beginning][]. - -This entry corresponds to [post five in Trystan's tutorial][trystan-tut]. - -If you want to follow along, the code for the series is [on Bitbucket][bb] and -[on GitHub][gh]. Update to the `entry-05` tag to see the code as it stands -after this post. - -Also, I live streamed myself writing the code that this entry is based on. You -can view the recordings [on twitch.tv](http://www.twitch.tv/stevelosh/), though -as I write this the video links are stuck in an infinite HTTP redirect loop. -Perhaps they will be fixed eventually. - -Finally, I've started hanging out in `##cavesofclojure` on Freenode if you have -questions. I may or may not be around at any given point. - -[the beginning]: /blog/2012/07/caves-of-clojure-01/ -[trystan-tut]: http://trystans.blogspot.com/2011/09/roguelike-tutorial-05-stationary.html -[bb]: http://bitbucket.org/sjl/caves/ -[gh]: http://github.com/sjl/caves/ - -[TOC] - -Summary -------- - -In Trystan's fifth post he adds three things: - -* A stationary monster -* Attacking -* A growing mechanic for the monster - -I'm going to add all three of those too, though I'll be doing it in the -entities/aspects fashion of the previous post. - -Multiple Entities ------------------ - -First thing's first: I need to change a bit of code around to account for having -multiple entities instead of just a single player. - -At the end of the previous post the player was stored in the world directly, -meaning the `game` object looked like this: - - :::clojure - {:uis [...] - :world {:player {...} - :tiles [...]}} - -I decided to remove some (but not all) of the special casing for the player and -make them an entity like any other. The `game` object is now structured like -this: - - :::clojure - {:uis [...] - :world {:entities {:player {...}} - :tiles [...]}} - -Notice that the player has been moved into an `:entities` map, keyed by its id -(which is still the special-cased `:player`). I updated anywhere that needed to -change to account for this, and made sure it still worked. This was pretty easy -to do by just searching for `:player`, as the codebase is still small. - -Lichens -------- - -Now it's time to actually add another type of entity. I play a lot of Nethack, -so naturally I decided to make it a simple lichen. I added -`entities/lichen.clj`: - - :::clojure - (ns caves.entities.lichen - (:use [caves.entities.core :only [Entity get-id]])) - - - (defrecord Lichen [id glyph location]) - - (defn make-lichen [location] - (->Lichen (get-id) "F" location)) - - - (extend-type Lichen Entity - (tick [this world] - (if (should-grow) - world))) - -Like the `Player`, `Lichen`s implement the `Entity` protocol. For now they -don't do anything special during a tick. - -You may have noticed the new `get-id` function. Entities must have IDs so I can -get them in and out of the entity map. The player has a special ID of -`:player`, but I needed a way to get a unique ID for other entities. - -The simplest way I could think of was to use a simple counter over in -`entities/core.clj`: - - :::clojure - (ns caves.entities.core) - - - (def ids (ref 0)) - - (defprotocol Entity - (tick [this world] - "Update the world to handle the passing of a tick for this entity.")) - - - (defn get-id [] - (dosync - (let [id @ids] - (alter ids inc) - id))) - -Not the prettiest solution, but it works. I might switch to a UUID library or -something in the future, but this'll do for now. - -Populating the World --------------------- - -Unlike the `make-player` function, `make-lichen` takes a location directly -instead of trying to find an empty space for itself in the world. I figured it -was better to not have entities deciding where they emerge in the world all the -time! I went ahead and refactored `make-player` to act like this as well. - -During this coding session I wasn't actually running and playing the full game -through as much as I should have been. I think as I get more and more of the -basic structure of the game in place I'll be able to do this more. Up to now -I've been doing large, sweeping refactorings that touch many different pieces of -code and break everything until they're finished. - -Anyway, back to the world. I need a way to spawn some lichens in the world, so -I edited the `reset-game` function in `input.clj`: - - :::clojure - (defn add-lichen [world] - (let [{:as lichen :keys [id]} (make-lichen (find-empty-tile world))] - (assoc-in world [:entities id] lichen))) - - (defn populate-world [world] - (let [world (assoc-in world [:entities :player] - (make-player (find-empty-tile world))) - world (nth (iterate add-lichen world) 30)] - world)) - - (defn reset-game [game] - (let [fresh-world (random-world)] - (-> game - (assoc :world fresh-world) - (update-in [:world] populate-world) - (assoc :uis [(->UI :play)])))) - -It should be pretty easy to read. `add-lichen` adds a new lichen to an empty -tile. `populate-world` takes a world and adds a player, then 30 lichens. - -This is getting to be a bit much to keep in `input.clj`. I'll probably pull -this out into a separate file soon. - -Drawing the Entities --------------------- - -So now the lichens are part of the world, but I still need to draw them on the -screen. I split the `draw-player` function in `drawing.clj` into two separate -functions: - - :::clojure - (defn draw-entity [screen start-x start-y {:keys [location glyph color]}] - (let [[entity-x entity-y] location - x (- entity-x start-x) - y (- entity-y start-y)] - (s/put-string screen x y glyph {:fg color}))) - - (defn highlight-player [screen start-x start-y player] - (let [[player-x player-y] (:location player) - x (- player-x start-x) - y (- player-y start-y)] - (s/move-cursor screen x y))) - -And then I use those in the main `draw-ui` function for the `:play` UI: - - :::clojure - (defmethod draw-ui :play [ui game screen] - (let [world (:world game) - {:keys [tiles entities]} world - player (:player entities) - [cols rows] screen-size - vcols cols - vrows (dec rows) - [start-x start-y end-x end-y] (get-viewport-coords game (:location player) vcols vrows)] - (draw-world screen vrows vcols start-x start-y end-x end-y tiles) - (doseq [entity (vals entities)] - (draw-entity screen start-x start-y entity)) - (draw-hud screen game start-x start-y) - (highlight-player screen start-x start-y player))) - -Long but straightforward. I'm going to be cleaning this part of the code up -very soon, as I've just added a bunch of really useful stuff to the -[clojure-lanterna][] library that will let me delete a bunch of fiddly code -here. - -[clojure-lanterna]: http://sjl.bitbucket.org/clojure-lanterna/ - -If you're particularly eagle-eyed you might have noticed this new `color` -attribute that seems to be a part of entities. I didn't realize I had forgotten -to specify colors until I actually wrote this bit of code. Once I did I went -back and added the field to the `Player` and `Lichen` records, as well as -`make-player` and `make-lichen`. - -Now the lichens appear on the screen! - -![Screenshot](/media/images{{ parent_url }}/caves-05-01.png) - -Movement --------- - -At this point the lichens are on the screen but the player can walk straight -through them. I took care of that by first creating a few helper functions in -`world.clj`: - - :::clojure - (defn get-entity-at [world coord] - (first (filter #(= coord (:location %)) - (vals (:entities world))))) - - (defn is-empty? [world coord] - (and (#{:floor} (get-tile-kind world coord)) - (not (get-entity-at world coord)))) - -They'll handle the grunt world of traversing the `world` data structure. Then -I updated the player's `can-move?` function: - - :::clojure - (extend-type Player Mobile - (move ...) - (can-move? [this world dest] - (is-empty? world dest))) - -Previously `can-move` checked the world's tile itself -- now it delegates to -a basic helper function instead. I have a feeling a lot of things are going to -need to use the idea of "empty tiles" so this function will probably get a lot -of mileage. - -Now the player can't walk through fungus. Great. - -Killing -------- - -It's time to give the player a way to cut the lichens into little licheny bits. -I implemented this with another pair of aspects: `Attacker` and `Destructible`. - -`Attacker` should be implemented by anything that can attack other things: - - :::clojure - (ns caves.entities.aspects.attacker) - - (defprotocol Attacker - (attack [this world target] - "Attack the target.")) - -`Destructible` should be implemented by anything that can "take damage and go -away once it takes enough": - - :::clojure - (ns caves.entities.aspects.destructible) - - (defprotocol Destructible - (take-damage [this world damage] - "Take the given amount of damage and update the world appropriately.")) - -Lichens will be `Destructible` (for now the player will remain invincible): - - :::clojure - (extend-type Lichen Destructible - (take-damage [{:keys [id] :as this} world damage] - (let [damaged-this (update-in this [:hp] - damage)] - (if-not (pos? (:hp damaged-this)) - (update-in world [:entities] dissoc id) - (update-in world [:entities id] assoc damaged-this))))) - -The logic here is pretty basic. When a `Destructible` entity takes some damage, -first its hit points are updated. If they wind up to be zero or fewer, the -entity gracefully removes itself from the world. - -I have a feeling there's a more elegant way to write the updatey bits of that -function. If you've got suggestions please let me know. - -If I'm going to be basing damage on the entity's `:hp` attribute they'd better -have one! I added a simple `:hp` of `1` to lichens: - - :::clojure - (defrecord Lichen [id glyph color location hp]) - - (defn make-lichen [location] - (->Lichen (get-id) "F" :green location 1)) - -Next I added the corresponding implementation of `Attacker` to the `Player` (for -now lichens can't strike back): - - :::clojure - (extend-type Player Attacker - (attack [this world target] - {:pre [(satisfies? Destructible target)]} - (let [damage 1] - (take-damage target world damage)))) - -Again, a very basic system for the moment: all attacks do one damage. Lichens -only have one hit point, so this will kill them instantly. - -Notice the precondition here: an attacker can attack something if and only if -it's something that satisfies the `Destructible` protocol. - -Instead of doing something like checking if the target has `:hp` I simply check -if it's `Destructible`. This opens the door for things that don't necessarily -use hit points, like a monster whose mana and hit points are a single number. - -Finally, I need to hook up the attacking functionality in the `move-player` -helper function: - - :::clojure - (defn move-player [world dir] - (let [player (get-in world [:entities :player]) - target (destination-coords (:location player) dir) - entity-at-target (get-entity-at world target)] - (cond - entity-at-target (attack player world entity-at-target) - (can-move? player world target) (move player world target) - (can-dig? player world target) (dig player world target) - :else world))) - -This once again overloads the `hjkl` keys, so now the player will attack -a monster when they try to move into it. Otherwise the player will move or dig -as before. - -Growing Lichens ---------------- - -Now for the last part of Trystan's post. Lichens should have a chance of -spreading slowly every turn. Unlike Trystan, I'm not going to limit the number -of times the lichen can spread, so the player will need to use their newfound -attacking ability if they want to stem the tide of invading fungus! - -This turned out to be surprisingly painless: - - :::clojure - (defn should-grow [] - (< (rand) 0.01)) - - (defn grow [lichen world] - (if-let [target (find-empty-neighbor world (:location lichen))] - (let [new-lichen (make-lichen target)] - (assoc-in world [:entities (:id new-lichen)] new-lichen)) - world)) - - (extend-type Lichen Entity - (tick [this world] - (if (should-grow) - (grow this world) - world))) - -Every tick, the lichen has a one percent chance to spread to an empty -neighboring tile. If there are no empty neighboring tiles, it can't spread. - -The `find-empty-neighbor` function is new, and located in `world.clj`: - - :::clojure - (defn find-empty-neighbor [world coord] - (let [candidates (filter #(is-empty? world %) (neighbors coord))] - (when (seq candidates) - (rand-nth candidates)))) - -It uses `neighbors`, which is another function I created after a quick refactor -of `coords.clj`: - - :::clojure - (ns caves.coords) - - (def directions - {:w [-1 0] - :e [1 0] - :n [0 -1] - :s [0 1] - :nw [-1 -1] - :ne [1 -1] - :sw [-1 1] - :se [1 1]}) - - (defn offset-coords - "Offset the starting coordinate by the given amount, returning the result coordinate." - [[x y] [dx dy]] - [(+ x dx) (+ y dy)]) - - (defn dir-to-offset - "Convert a direction to the offset for moving 1 in that direction." - [dir] - (directions dir)) - - (defn destination-coords - "Take an origin's coords and a direction and return the destination's coords." - [origin dir] - (offset-coords origin (dir-to-offset dir))) - - (defn neighbors - "Return the coordinates of all neighboring squares of the given coord." - [origin] - (map offset-coords (vals directions) (repeat origin))) - -Nothing too crazy here. The small, composable functions build on top of each -other to create more interesting ones. - -But there's one thing left to do, which is actually `tick` entities in the main -game loop in `core.clj`: - - :::clojure - (defn tick-entity [world entity] - (tick entity world)) - - (defn tick-all [world] - (reduce tick-entity world (vals (:entities world)))) - - (defn run-game [game screen] - (loop [{:keys [input uis] :as game} game] - (when-not (empty? uis) - (draw-game game screen) - (if (nil? input) - (recur (get-input (update-in game [:world] tick-all) screen)) - (recur (process-input (dissoc game :input) input)))))) - -Notice how the `tick-all` function reduces over the values in the entities map. -Maps aren't deterministically ordered (or at least they're not *guaranteed* to -be), so this means that our entities may process their ticks in a different -order each turn. - -I think I'm okay with that. Yes, it means that ticking the world isn't going to -be a pure function, but it won't be pure no matter what since we're going to -have random numbers involved in attacking and damage soon enough. - -Results -------- - -All-in-all it took roughly an hour and a half to code the stuff in this entry. -This might sound like a lot, but remember what was added: - -* The entire concept of "multiple entities in a map". -* Support for drawing arbitrary entities on the map. -* A new creature. -* Entities blocking movement of others. -* A rudimentary attacking gameplay mechanic. -* A rudimentary killing mechanic. -* Ticking entities. -* Growing/spreading of creatures. -* Lots of refactoring and helper functions. - -Not too bad! - -You can view the code [on GitHub][result-code] if you want to see the end -result. - -[result-code]: https://github.com/sjl/caves/tree/entry-05/src/caves - -And now some screenshots of our hero cutting a swath through some fungus! - -![Screenshot](/media/images{{ parent_url }}/caves-05-02.png) - -![Screenshot](/media/images{{ parent_url }}/caves-05-03.png) - -I'll be moving on to Trystan's sixth post soon, but before that I'm going to -have another interlude where I explain some quick refactoring and then work -a bit of the blackest magic in Clojure: a non-trivial macro. - -{% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/07/caves-of-clojure-05.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2012/07/caves-of-clojure-05.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,496 @@ ++++ +title = "The Caves of Clojure: Part 5" +snip = "Fungus and more." +date = 2012-07-13T10:55:00Z +draft = false + ++++ + +This post is part of an ongoing series. If you haven't already done so, you +should probably start at [the beginning][]. + +This entry corresponds to [post five in Trystan's tutorial][trystan-tut]. + +If you want to follow along, the code for the series is [on Bitbucket][bb] and +[on GitHub][gh]. Update to the `entry-05` tag to see the code as it stands +after this post. + +Also, I live streamed myself writing the code that this entry is based on. You +can view the recordings [on twitch.tv](http://www.twitch.tv/stevelosh/), though +as I write this the video links are stuck in an infinite HTTP redirect loop. +Perhaps they will be fixed eventually. + +Finally, I've started hanging out in `##cavesofclojure` on Freenode if you have +questions. I may or may not be around at any given point. + +[the beginning]: /blog/2012/07/caves-of-clojure-01/ +[trystan-tut]: http://trystans.blogspot.com/2011/09/roguelike-tutorial-05-stationary.html +[bb]: http://bitbucket.org/sjl/caves/ +[gh]: http://github.com/sjl/caves/ + +{{% toc %}} + +Summary +------- + +In Trystan's fifth post he adds three things: + +* A stationary monster +* Attacking +* A growing mechanic for the monster + +I'm going to add all three of those too, though I'll be doing it in the +entities/aspects fashion of the previous post. + +Multiple Entities +----------------- + +First thing's first: I need to change a bit of code around to account for having +multiple entities instead of just a single player. + +At the end of the previous post the player was stored in the world directly, +meaning the `game` object looked like this: + +```clojure +{:uis [...] + :world {:player {...} + :tiles [...]}} +``` + +I decided to remove some (but not all) of the special casing for the player and +make them an entity like any other. The `game` object is now structured like +this: + +```clojure +{:uis [...] + :world {:entities {:player {...}} + :tiles [...]}} +``` + +Notice that the player has been moved into an `:entities` map, keyed by its id +(which is still the special-cased `:player`). I updated anywhere that needed to +change to account for this, and made sure it still worked. This was pretty easy +to do by just searching for `:player`, as the codebase is still small. + +Lichens +------- + +Now it's time to actually add another type of entity. I play a lot of Nethack, +so naturally I decided to make it a simple lichen. I added +`entities/lichen.clj`: + +```clojure +(ns caves.entities.lichen + (:use [caves.entities.core :only [Entity get-id]])) + + +(defrecord Lichen [id glyph location]) + +(defn make-lichen [location] + (->Lichen (get-id) "F" location)) + + +(extend-type Lichen Entity + (tick [this world] + (if (should-grow) + world))) +``` + +Like the `Player`, `Lichen`s implement the `Entity` protocol. For now they +don't do anything special during a tick. + +You may have noticed the new `get-id` function. Entities must have IDs so I can +get them in and out of the entity map. The player has a special ID of +`:player`, but I needed a way to get a unique ID for other entities. + +The simplest way I could think of was to use a simple counter over in +`entities/core.clj`: + +```clojure +(ns caves.entities.core) + + +(def ids (ref 0)) + +(defprotocol Entity + (tick [this world] + "Update the world to handle the passing of a tick for this entity.")) + + +(defn get-id [] + (dosync + (let [id @ids] + (alter ids inc) + id))) +``` + +Not the prettiest solution, but it works. I might switch to a UUID library or +something in the future, but this'll do for now. + +Populating the World +-------------------- + +Unlike the `make-player` function, `make-lichen` takes a location directly +instead of trying to find an empty space for itself in the world. I figured it +was better to not have entities deciding where they emerge in the world all the +time! I went ahead and refactored `make-player` to act like this as well. + +During this coding session I wasn't actually running and playing the full game +through as much as I should have been. I think as I get more and more of the +basic structure of the game in place I'll be able to do this more. Up to now +I've been doing large, sweeping refactorings that touch many different pieces of +code and break everything until they're finished. + +Anyway, back to the world. I need a way to spawn some lichens in the world, so +I edited the `reset-game` function in `input.clj`: + +```clojure +(defn add-lichen [world] + (let [{:as lichen :keys [id]} (make-lichen (find-empty-tile world))] + (assoc-in world [:entities id] lichen))) + +(defn populate-world [world] + (let [world (assoc-in world [:entities :player] + (make-player (find-empty-tile world))) + world (nth (iterate add-lichen world) 30)] + world)) + +(defn reset-game [game] + (let [fresh-world (random-world)] + (-> game + (assoc :world fresh-world) + (update-in [:world] populate-world) + (assoc :uis [(->UI :play)])))) +``` + +It should be pretty easy to read. `add-lichen` adds a new lichen to an empty +tile. `populate-world` takes a world and adds a player, then 30 lichens. + +This is getting to be a bit much to keep in `input.clj`. I'll probably pull +this out into a separate file soon. + +Drawing the Entities +-------------------- + +So now the lichens are part of the world, but I still need to draw them on the +screen. I split the `draw-player` function in `drawing.clj` into two separate +functions: + +```clojure +(defn draw-entity [screen start-x start-y {:keys [location glyph color]}] + (let [[entity-x entity-y] location + x (- entity-x start-x) + y (- entity-y start-y)] + (s/put-string screen x y glyph {:fg color}))) + +(defn highlight-player [screen start-x start-y player] + (let [[player-x player-y] (:location player) + x (- player-x start-x) + y (- player-y start-y)] + (s/move-cursor screen x y))) +``` + +And then I use those in the main `draw-ui` function for the `:play` UI: + +```clojure +(defmethod draw-ui :play [ui game screen] + (let [world (:world game) + {:keys [tiles entities]} world + player (:player entities) + [cols rows] screen-size + vcols cols + vrows (dec rows) + [start-x start-y end-x end-y] (get-viewport-coords game (:location player) vcols vrows)] + (draw-world screen vrows vcols start-x start-y end-x end-y tiles) + (doseq [entity (vals entities)] + (draw-entity screen start-x start-y entity)) + (draw-hud screen game start-x start-y) + (highlight-player screen start-x start-y player))) +``` + +Long but straightforward. I'm going to be cleaning this part of the code up +very soon, as I've just added a bunch of really useful stuff to the +[clojure-lanterna][] library that will let me delete a bunch of fiddly code +here. + +[clojure-lanterna]: http://sjl.bitbucket.org/clojure-lanterna/ + +If you're particularly eagle-eyed you might have noticed this new `color` +attribute that seems to be a part of entities. I didn't realize I had forgotten +to specify colors until I actually wrote this bit of code. Once I did I went +back and added the field to the `Player` and `Lichen` records, as well as +`make-player` and `make-lichen`. + +Now the lichens appear on the screen! + +![Screenshot](/media/images/blog/2012/07/caves-05-01.png) + +Movement +-------- + +At this point the lichens are on the screen but the player can walk straight +through them. I took care of that by first creating a few helper functions in +`world.clj`: + +```clojure +(defn get-entity-at [world coord] + (first (filter #(= coord (:location %)) + (vals (:entities world))))) + +(defn is-empty? [world coord] + (and (#{:floor} (get-tile-kind world coord)) + (not (get-entity-at world coord)))) +``` + +They'll handle the grunt world of traversing the `world` data structure. Then +I updated the player's `can-move?` function: + +```clojure +(extend-type Player Mobile + (move ...) + (can-move? [this world dest] + (is-empty? world dest))) +``` + +Previously `can-move` checked the world's tile itself — now it delegates to +a basic helper function instead. I have a feeling a lot of things are going to +need to use the idea of "empty tiles" so this function will probably get a lot +of mileage. + +Now the player can't walk through fungus. Great. + +Killing +------- + +It's time to give the player a way to cut the lichens into little licheny bits. +I implemented this with another pair of aspects: `Attacker` and `Destructible`. + +`Attacker` should be implemented by anything that can attack other things: + +```clojure +(ns caves.entities.aspects.attacker) + +(defprotocol Attacker + (attack [this world target] + "Attack the target.")) +``` + +`Destructible` should be implemented by anything that can "take damage and go +away once it takes enough": + +```clojure +(ns caves.entities.aspects.destructible) + +(defprotocol Destructible + (take-damage [this world damage] + "Take the given amount of damage and update the world appropriately.")) +``` + +Lichens will be `Destructible` (for now the player will remain invincible): + +```clojure +(extend-type Lichen Destructible + (take-damage [{:keys [id] :as this} world damage] + (let [damaged-this (update-in this [:hp] - damage)] + (if-not (pos? (:hp damaged-this)) + (update-in world [:entities] dissoc id) + (update-in world [:entities id] assoc damaged-this))))) +``` + +The logic here is pretty basic. When a `Destructible` entity takes some damage, +first its hit points are updated. If they wind up to be zero or fewer, the +entity gracefully removes itself from the world. + +I have a feeling there's a more elegant way to write the updatey bits of that +function. If you've got suggestions please let me know. + +If I'm going to be basing damage on the entity's `:hp` attribute they'd better +have one! I added a simple `:hp` of `1` to lichens: + +```clojure +(defrecord Lichen [id glyph color location hp]) + +(defn make-lichen [location] + (->Lichen (get-id) "F" :green location 1)) +``` + +Next I added the corresponding implementation of `Attacker` to the `Player` (for +now lichens can't strike back): + +```clojure +(extend-type Player Attacker + (attack [this world target] + {:pre [(satisfies? Destructible target)]} + (let [damage 1] + (take-damage target world damage)))) +``` + +Again, a very basic system for the moment: all attacks do one damage. Lichens +only have one hit point, so this will kill them instantly. + +Notice the precondition here: an attacker can attack something if and only if +it's something that satisfies the `Destructible` protocol. + +Instead of doing something like checking if the target has `:hp` I simply check +if it's `Destructible`. This opens the door for things that don't necessarily +use hit points, like a monster whose mana and hit points are a single number. + +Finally, I need to hook up the attacking functionality in the `move-player` +helper function: + +```clojure +(defn move-player [world dir] + (let [player (get-in world [:entities :player]) + target (destination-coords (:location player) dir) + entity-at-target (get-entity-at world target)] + (cond + entity-at-target (attack player world entity-at-target) + (can-move? player world target) (move player world target) + (can-dig? player world target) (dig player world target) + :else world))) +``` + +This once again overloads the `hjkl` keys, so now the player will attack +a monster when they try to move into it. Otherwise the player will move or dig +as before. + +Growing Lichens +--------------- + +Now for the last part of Trystan's post. Lichens should have a chance of +spreading slowly every turn. Unlike Trystan, I'm not going to limit the number +of times the lichen can spread, so the player will need to use their newfound +attacking ability if they want to stem the tide of invading fungus! + +This turned out to be surprisingly painless: + +```clojure +(defn should-grow [] + (< (rand) 0.01)) + +(defn grow [lichen world] + (if-let [target (find-empty-neighbor world (:location lichen))] + (let [new-lichen (make-lichen target)] + (assoc-in world [:entities (:id new-lichen)] new-lichen)) + world)) + +(extend-type Lichen Entity + (tick [this world] + (if (should-grow) + (grow this world) + world))) +``` + +Every tick, the lichen has a one percent chance to spread to an empty +neighboring tile. If there are no empty neighboring tiles, it can't spread. + +The `find-empty-neighbor` function is new, and located in `world.clj`: + +```clojure +(defn find-empty-neighbor [world coord] + (let [candidates (filter #(is-empty? world %) (neighbors coord))] + (when (seq candidates) + (rand-nth candidates)))) +``` + +It uses `neighbors`, which is another function I created after a quick refactor +of `coords.clj`: + +```clojure +(ns caves.coords) + +(def directions + {:w [-1 0] + :e [1 0] + :n [0 -1] + :s [0 1] + :nw [-1 -1] + :ne [1 -1] + :sw [-1 1] + :se [1 1]}) + +(defn offset-coords + "Offset the starting coordinate by the given amount, returning the result coordinate." + [[x y] [dx dy]] + [(+ x dx) (+ y dy)]) + +(defn dir-to-offset + "Convert a direction to the offset for moving 1 in that direction." + [dir] + (directions dir)) + +(defn destination-coords + "Take an origin's coords and a direction and return the destination's coords." + [origin dir] + (offset-coords origin (dir-to-offset dir))) + +(defn neighbors + "Return the coordinates of all neighboring squares of the given coord." + [origin] + (map offset-coords (vals directions) (repeat origin))) +``` + +Nothing too crazy here. The small, composable functions build on top of each +other to create more interesting ones. + +But there's one thing left to do, which is actually `tick` entities in the main +game loop in `core.clj`: + +```clojure +(defn tick-entity [world entity] + (tick entity world)) + +(defn tick-all [world] + (reduce tick-entity world (vals (:entities world)))) + +(defn run-game [game screen] + (loop [{:keys [input uis] :as game} game] + (when-not (empty? uis) + (draw-game game screen) + (if (nil? input) + (recur (get-input (update-in game [:world] tick-all) screen)) + (recur (process-input (dissoc game :input) input)))))) +``` + +Notice how the `tick-all` function reduces over the values in the entities map. +Maps aren't deterministically ordered (or at least they're not *guaranteed* to +be), so this means that our entities may process their ticks in a different +order each turn. + +I think I'm okay with that. Yes, it means that ticking the world isn't going to +be a pure function, but it won't be pure no matter what since we're going to +have random numbers involved in attacking and damage soon enough. + +Results +------- + +All-in-all it took roughly an hour and a half to code the stuff in this entry. +This might sound like a lot, but remember what was added: + +* The entire concept of "multiple entities in a map". +* Support for drawing arbitrary entities on the map. +* A new creature. +* Entities blocking movement of others. +* A rudimentary attacking gameplay mechanic. +* A rudimentary killing mechanic. +* Ticking entities. +* Growing/spreading of creatures. +* Lots of refactoring and helper functions. + +Not too bad! + +You can view the code [on GitHub][result-code] if you want to see the end +result. + +[result-code]: https://github.com/sjl/caves/tree/entry-05/src/caves + +And now some screenshots of our hero cutting a swath through some fungus! + +![Screenshot](/media/images/blog/2012/07/caves-05-02.png) + +![Screenshot](/media/images/blog/2012/07/caves-05-03.png) + +I'll be moving on to Trystan's sixth post soon, but before that I'm going to +have another interlude where I explain some quick refactoring and then work +a bit of the blackest magic in Clojure: a non-trivial macro. + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/07/caves-of-clojure-06.html --- a/content/blog/2012/07/caves-of-clojure-06.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,388 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "The Caves of Clojure: Part 6" - snip: "Real combat and messages." - created: 2012-07-30 09:50:00 - %} - -{% block article %} - -This post is part of an ongoing series. If you haven't already done so, you -should probably start at [the beginning][]. - -This entry corresponds to [post six in Trystan's tutorial][trystan-tut]. - -If you want to follow along, the code for the series is [on Bitbucket][bb] and -[on GitHub][gh]. Update to the `entry-06` tag to see the code as it stands -after this post. - -Sorry for the long wait for this entry. I've been working on lots of other -stuff and haven't had a lot of time to write. Hopefully I can get back to it -more often! - -[the beginning]: /blog/2012/07/caves-of-clojure-01/ -[trystan-tut]: http://trystans.blogspot.com/2011/09/roguelike-tutorial-06-hitpoints-combat.html -[bb]: http://bitbucket.org/sjl/caves/ -[gh]: http://github.com/sjl/caves/ - -[TOC] - -Summary -------- - -In Trystan's sixth post he adds a combat system and messaging infrastructure. -Once again I'm following his lead and implementing the same things, but in the -entity/aspect way of doing things. - -As usual I ended up refactoring a few things, which I'll briefly cover first. - -Refactoring ------------ - -So far, the functions entities implement to fulfill aspects have looked like -this: - - :::clojure - (defaspect Digger - (dig [this world dest] - ...) - (can-dig? [this world dest] - ...)) - -The entity has to be the first argument, because that's how protocols work. -I don't have any flexibility there. I originally made the world always be the -second argument, but it turns out that it's more convenient to make the world -the *last* argument. - -To see why, imagine we want to allow players to dig and move at the same time, -instead of forcing them to be separate actions. Updating the world might look -like this: - - :::clojure - (let [new-world (dig player world dest) - new-world (move player world dest)] - new-world) - -You could make this one specific case a bit prettier, but in general chaining -together world-modifying actions is going to be a pain. If I change the aspect -functions to take the player, then other args, and *then* the world, I can use -`->>` to chain actions: - - :::clojure - (->> world - (dig player dest) - (move player dest)) - -Much cleaner! I went ahead and switched all the aspect functions to use this -new scheme. - -I also did some other minor refactoring. You can look through the changesets if -you're really curious. - -Attacking and Defending ------------------------ - -Instead of simply killing everything in one hit, I'm now going to give some -creatures a bit of hp. I also added a `:max-hp` attribute, since I'll likely -need that in the future. Here's a sample of what the `Bunny` creation function -looks like: - - :::clojure - (defn make-bunny [location] - (map->Bunny {:id (get-id) - :name "bunny" - :glyph "v" - :color :yellow - :location location - :hp 4 - :max-hp 4})) - -I started using the `map->Foo` record constructors because the `->Foo` versions -that relied on positional arguments started getting hard to read. - -Bunnies have 4 hp. It'd be trivial to randomize this in the future, but for now -I'll stick with a simple number. - -Trystan uses a simple attack and defense system. I toyed with the idea of using -a different one (like Brogue's) but figured I should stick to his tutorial when -there's no clear reason not to. - -Trystan's system needs attack and defense values, so I added functions to the -`Attacker` and `Destructible` aspects to retrieve these: - - :::clojure - (defaspect Attacker - (attack [this target world] - ...) - (attack-value [this world] - (get this :attack 1))) - - (defaspect Destructible - (take-damage [this damage world] - ...) - (defense-value [this world] - (get this :defense 0))) - -The default attack value is `1`. If an entity has an `:attack` attribute that -will be used instead. Or the entity could provide a completely custom version -of `attack-value` (e.g.: werewolves could have a larger attack if there's a full -moon in the game or something). Defense values work the same way. - -Now that I'm starting to actually use HP I'll display it in the bottom row of -info on the screen: - - :::clojure - (defn draw-hud [screen game] - (let [hud-row (dec (second (s/get-size screen))) - player (get-in game [:world :entities :player]) - {:keys [location hp max-hp]} player - [x y] location - info (str "hp [" hp "/" max-hp "]") - info (str info " loc: [" x "-" y "]")] - (s/put-string screen 0 hud-row info))) - -The bottom row of the screen now looks like: "hp [20/20] loc: [82-103]". I'll -probably get rid of the loc soon, but for now it won't hurt to keep it onscreen. - -Now for the damage calculation. I added a little helper function to take care -of this in `attacker.clj`: - - :::clojure - (defn get-damage [attacker target world] - (let [attack (attack-value attacker world) - defense (defense-value target world) - max-damage (max 0 (- attack defense)) - damage (inc (rand-int max-damage))] - damage)) - -This matches what Trystan does. In a nutshell, the damage done is: "If defense -is higher than attack, then 1. Otherwise, a random number between 1 and (attack -- defense)." - -I kept it separate from the `attack` function so that an entity can override -attack without having to reimplement this logic. - -The `attack` default implementation needs to use this new damage calculator: - - :::clojure - (defaspect Attacker - (attack [this target world] - {:pre [(satisfies? Destructible target)]} - (let [damage (get-damage this target world)] - (take-damage target damage))) - (attack-value [this world] - (get this :attack 1))) - -`Destructible` already handles reducing HP appropriately. The only thing left -is to give my entities some non-1 attack, defense, and/or hp values. For now -I used the following values: - -* Bunnies have 4 HP, default defense. -* Lichens have 6 HP, default defense. -* Silverfish have 15 HP, default defense. -* Players have 40 HP, 10 attack, default defense. - -These are really just placeholder numbers until I add the ability for monsters -to attack back. Once I do that I'll be able to play the game a bit and -determine if it's too easy or hard. - -Messaging ---------- - -Now the player can attack things and it may take a few swings to kill them. The -problem is that there's no feedback while this is going on, so it's hard to tell -that you're actually doing damage until the monster dies. - -A messaging system will let me display informational messages to give the player -some feedback. I decided to implement this like everything else: as an aspect. - -An entity that implements the `Receiver` protocol will be able to receive -messages. Here's `entities/aspects/receiver.clj`: - - :::clojure - (ns caves.entities.aspects.receiver - (:use [caves.entities.core :only [defaspect]] - [caves.world :only [get-entities-around]])) - - (defaspect Receiver - (receive-message [this message world] - (update-in world [:entities (:id this) :messages] conj message))) - - (defn send-message [entity message args world] - (if (satisfies? Receiver entity) - (receive-message entity (apply format message args) world) - world)) - -I've got a helper function `send-message` which is what entities will use to -send messages, instead of performing the `(satisfies? Receiver entity)` check -themselves every time. If the entity they're sending the message to isn't -a `Receiver` it will simply drop the message on the floor by returning the world -unchanged. It also handles formatting the message string for them. - -The default `receive-message` simply appends the message to a `:messages` -attribute in the entity. - -I have a lot of ideas about extending this system in the future, but for now -I'll just keep it simple to match Trystan's. - -Now I need to send some messages. The most obvious place to do this is when -something attacks something else, so I updated the `Attacker` aspect once more: - - - :::clojure - (defaspect Attacker - (attack [this target world] - {:pre [(satisfies? Destructible target)]} - (let [damage (get-damage this target world)] - (->> world - (take-damage target damage) - (send-message this "You strike the %s for %d damage!" - [(:name target) damage]) - (send-message target "The %s strikes you for %d damage!" - [(:name this) damage])))) - (attack-value [this world] - (get this :attack 1))) - -This is starting to get a little crowded. If I need to do much more in here -I'll refactor some stuff out into helper functions. But for now it's still -readable. - -Here you can see how making world-altering functions take the world as the last -argument pays off by letting me use `->>` to chain together actions. - -I also added a `:name` attribute to entities so I can say "You strike the bunny" -instead of "You strike the v". Nothing too special there. - -Finally, I need a way to notify nearby entities when something happens. Here's -what I came up with: - - :::clojure - (defn send-message-nearby [coord message world] - (let [entities (get-entities-around world coord 7) - sm (fn [world entity] - (send-message entity message [] world))] - (reduce sm world entities))) - -First I grab all the entities within 7 squares of the message coordinate. Then -I create a little helper function called `sm` that wraps `send-message`. It -will take a world and an entity and return the modified world. I use `reduce` -here to iterate over the entities and send the message to each one. It's -a pretty way of handling that looping. - -The `get-entities-around` function is new: - - :::clojure - (defn get-entities-around - ([world coord] (get-entities-around world coord 1)) - ([world coord radius] - (filter #(<= (radial-distance coord (:location %)) - radius) - (vals (:entities world))))) - -It looks through all the entities in the world and returns a sequence of those -whose "radial" distance is less than or equal to the given radius. The "radial -distance" (also called the "king's move" distance by some) looks like this: - - :::text - 3333333 - 3222223 - 3211123 - 3210123 - 3211123 - 3222223 - 3333333 - -And the function: - - :::clojure - (defn radial-distance - "Return the radial distance between two points." - [[x1 y1] [x2 y2]] - (max (abs (- x1 x2)) - (abs (- y1 y2)))) - -I may end up needing to modify this `send-message-nearby` function to be a bit -more powerful in the future. Trystan's version modifies the verbs and such. -For now this is good enough for me. - -Now I can make lichens notify nearby creatures when they grow: - - :::clojure - (defn grow [{:keys [location]} world] - (if-let [target (find-empty-neighbor world location)] - (let [new-lichen (make-lichen target) - world (assoc-in world [:entities (:id new-lichen)] new-lichen) - world (send-message-nearby location "The lichen grows." world)] - world) - world)) - -The last step is to actually display these messages to the player. I added -a `draw-messages` function to the `ui/drawing.clj` file: - - :::clojure - (defn draw-messages [screen messages] - (doseq [[i msg] (enumerate messages)] - (s/put-string screen 0 i msg {:fg :black :bg :white}))) - -And then I modified the main `draw-ui` function for `:play` UIs to draw the -messages on top of the map: - - :::clojure - (defmethod draw-ui :play [ui game screen] - (let [world (:world game) - {:keys [tiles entities]} world - player (:player entities) - [cols rows] (s/get-size screen) - vcols cols - vrows (dec rows) - origin (get-viewport-coords game (:location player) vcols vrows)] - (draw-world screen vrows vcols origin tiles) - (doseq [entity (vals entities)] - (draw-entity screen origin vrows vcols entity)) - (draw-hud screen game) - (draw-messages screen (:messages player)) - (highlight-player screen origin player))) - -This does mean that messages will cover a bit of the screen. For now I'll live -with that, but in the future it's something to fix. - -Finally we need to clear the message queue out periodically, otherwise it'll -grow until it covers the entire screen! I modified the main game loop in -`core.clj`: - - :::clojure - (defn clear-messages [game] - (assoc-in game [:world :entities :player :messages] nil)) - - (defn run-game [game screen] - (loop [{:keys [input uis] :as game} game] - (when (seq uis) - (if (nil? input) - (let [game (update-in game [:world] tick-all) - _ (draw-game game screen) - game (clear-messages game)] - (recur (get-input game screen))) - (recur (process-input (dissoc game :input) input)))))) - -Results -------- - -After fixing a few other bugs (you can read the changelog if you're interested) -I've now got a working combat system, and a messaging system so I can tell -what's going on: - -![Screenshot](/media/images{{ parent_url }}/caves-06-01.png) - -It's actually starting to feel like a real game now, instead of just a sandbox -where you can break things. - -You can view the code [on GitHub][result-code] if you want to see the end -result. - -[result-code]: https://github.com/sjl/caves/tree/entry-06/src/caves - -The next article will move on to Trystan's seventh post, which adds multiple -z-levels to the caves. - -{% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/07/caves-of-clojure-06.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2012/07/caves-of-clojure-06.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,403 @@ ++++ +title = "The Caves of Clojure: Part 6" +snip = "Real combat and messages." +date = 2012-07-30T09:50:00Z +draft = false + ++++ + +This post is part of an ongoing series. If you haven't already done so, you +should probably start at [the beginning][]. + +This entry corresponds to [post six in Trystan's tutorial][trystan-tut]. + +If you want to follow along, the code for the series is [on Bitbucket][bb] and +[on GitHub][gh]. Update to the `entry-06` tag to see the code as it stands +after this post. + +Sorry for the long wait for this entry. I've been working on lots of other +stuff and haven't had a lot of time to write. Hopefully I can get back to it +more often! + +[the beginning]: /blog/2012/07/caves-of-clojure-01/ +[trystan-tut]: http://trystans.blogspot.com/2011/09/roguelike-tutorial-06-hitpoints-combat.html +[bb]: http://bitbucket.org/sjl/caves/ +[gh]: http://github.com/sjl/caves/ + +{{% toc %}} + +Summary +------- + +In Trystan's sixth post he adds a combat system and messaging infrastructure. +Once again I'm following his lead and implementing the same things, but in the +entity/aspect way of doing things. + +As usual I ended up refactoring a few things, which I'll briefly cover first. + +Refactoring +----------- + +So far, the functions entities implement to fulfill aspects have looked like +this: + +```clojure +(defaspect Digger + (dig [this world dest] + ...) + (can-dig? [this world dest] + ...)) +``` + +The entity has to be the first argument, because that's how protocols work. +I don't have any flexibility there. I originally made the world always be the +second argument, but it turns out that it's more convenient to make the world +the *last* argument. + +To see why, imagine we want to allow players to dig and move at the same time, +instead of forcing them to be separate actions. Updating the world might look +like this: + +```clojure +(let [new-world (dig player world dest) + new-world (move player world dest)] + new-world) +``` + +You could make this one specific case a bit prettier, but in general chaining +together world-modifying actions is going to be a pain. If I change the aspect +functions to take the player, then other args, and *then* the world, I can use +`->>` to chain actions: + +```clojure +(->> world + (dig player dest) + (move player dest)) +``` + +Much cleaner! I went ahead and switched all the aspect functions to use this +new scheme. + +I also did some other minor refactoring. You can look through the changesets if +you're really curious. + +Attacking and Defending +----------------------- + +Instead of simply killing everything in one hit, I'm now going to give some +creatures a bit of hp. I also added a `:max-hp` attribute, since I'll likely +need that in the future. Here's a sample of what the `Bunny` creation function +looks like: + +```clojure +(defn make-bunny [location] + (map->Bunny {:id (get-id) + :name "bunny" + :glyph "v" + :color :yellow + :location location + :hp 4 + :max-hp 4})) +``` + +I started using the `map->Foo` record constructors because the `->Foo` versions +that relied on positional arguments started getting hard to read. + +Bunnies have 4 hp. It'd be trivial to randomize this in the future, but for now +I'll stick with a simple number. + +Trystan uses a simple attack and defense system. I toyed with the idea of using +a different one (like Brogue's) but figured I should stick to his tutorial when +there's no clear reason not to. + +Trystan's system needs attack and defense values, so I added functions to the +`Attacker` and `Destructible` aspects to retrieve these: + +```clojure +(defaspect Attacker + (attack [this target world] + ...) + (attack-value [this world] + (get this :attack 1))) + +(defaspect Destructible + (take-damage [this damage world] + ...) + (defense-value [this world] + (get this :defense 0))) +``` + +The default attack value is `1`. If an entity has an `:attack` attribute that +will be used instead. Or the entity could provide a completely custom version +of `attack-value` (e.g.: werewolves could have a larger attack if there's a full +moon in the game or something). Defense values work the same way. + +Now that I'm starting to actually use HP I'll display it in the bottom row of +info on the screen: + +```clojure +(defn draw-hud [screen game] + (let [hud-row (dec (second (s/get-size screen))) + player (get-in game [:world :entities :player]) + {:keys [location hp max-hp]} player + [x y] location + info (str "hp [" hp "/" max-hp "]") + info (str info " loc: [" x "-" y "]")] + (s/put-string screen 0 hud-row info))) +``` + +The bottom row of the screen now looks like: "hp [20/20] loc: [82-103]". I'll +probably get rid of the loc soon, but for now it won't hurt to keep it onscreen. + +Now for the damage calculation. I added a little helper function to take care +of this in `attacker.clj`: + +```clojure +(defn get-damage [attacker target world] + (let [attack (attack-value attacker world) + defense (defense-value target world) + max-damage (max 0 (- attack defense)) + damage (inc (rand-int max-damage))] + damage)) +``` + +This matches what Trystan does. In a nutshell, the damage done is: "If defense +is higher than attack, then 1. Otherwise, a random number between 1 and (attack +- defense)." + +I kept it separate from the `attack` function so that an entity can override +attack without having to reimplement this logic. + +The `attack` default implementation needs to use this new damage calculator: + +```clojure +(defaspect Attacker + (attack [this target world] + {:pre [(satisfies? Destructible target)]} + (let [damage (get-damage this target world)] + (take-damage target damage))) + (attack-value [this world] + (get this :attack 1))) +``` + +`Destructible` already handles reducing HP appropriately. The only thing left +is to give my entities some non-1 attack, defense, and/or hp values. For now +I used the following values: + +* Bunnies have 4 HP, default defense. +* Lichens have 6 HP, default defense. +* Silverfish have 15 HP, default defense. +* Players have 40 HP, 10 attack, default defense. + +These are really just placeholder numbers until I add the ability for monsters +to attack back. Once I do that I'll be able to play the game a bit and +determine if it's too easy or hard. + +Messaging +--------- + +Now the player can attack things and it may take a few swings to kill them. The +problem is that there's no feedback while this is going on, so it's hard to tell +that you're actually doing damage until the monster dies. + +A messaging system will let me display informational messages to give the player +some feedback. I decided to implement this like everything else: as an aspect. + +An entity that implements the `Receiver` protocol will be able to receive +messages. Here's `entities/aspects/receiver.clj`: + +```clojure +(ns caves.entities.aspects.receiver + (:use [caves.entities.core :only [defaspect]] + [caves.world :only [get-entities-around]])) + +(defaspect Receiver + (receive-message [this message world] + (update-in world [:entities (:id this) :messages] conj message))) + +(defn send-message [entity message args world] + (if (satisfies? Receiver entity) + (receive-message entity (apply format message args) world) + world)) +``` + +I've got a helper function `send-message` which is what entities will use to +send messages, instead of performing the `(satisfies? Receiver entity)` check +themselves every time. If the entity they're sending the message to isn't +a `Receiver` it will simply drop the message on the floor by returning the world +unchanged. It also handles formatting the message string for them. + +The default `receive-message` simply appends the message to a `:messages` +attribute in the entity. + +I have a lot of ideas about extending this system in the future, but for now +I'll just keep it simple to match Trystan's. + +Now I need to send some messages. The most obvious place to do this is when +something attacks something else, so I updated the `Attacker` aspect once more: + + +```clojure +(defaspect Attacker + (attack [this target world] + {:pre [(satisfies? Destructible target)]} + (let [damage (get-damage this target world)] + (->> world + (take-damage target damage) + (send-message this "You strike the %s for %d damage!" + [(:name target) damage]) + (send-message target "The %s strikes you for %d damage!" + [(:name this) damage])))) + (attack-value [this world] + (get this :attack 1))) +``` + +This is starting to get a little crowded. If I need to do much more in here +I'll refactor some stuff out into helper functions. But for now it's still +readable. + +Here you can see how making world-altering functions take the world as the last +argument pays off by letting me use `->>` to chain together actions. + +I also added a `:name` attribute to entities so I can say "You strike the bunny" +instead of "You strike the v". Nothing too special there. + +Finally, I need a way to notify nearby entities when something happens. Here's +what I came up with: + +```clojure +(defn send-message-nearby [coord message world] + (let [entities (get-entities-around world coord 7) + sm (fn [world entity] + (send-message entity message [] world))] + (reduce sm world entities))) +``` + +First I grab all the entities within 7 squares of the message coordinate. Then +I create a little helper function called `sm` that wraps `send-message`. It +will take a world and an entity and return the modified world. I use `reduce` +here to iterate over the entities and send the message to each one. It's +a pretty way of handling that looping. + +The `get-entities-around` function is new: + +```clojure +(defn get-entities-around + ([world coord] (get-entities-around world coord 1)) + ([world coord radius] + (filter #(<= (radial-distance coord (:location %)) + radius) + (vals (:entities world))))) +``` + +It looks through all the entities in the world and returns a sequence of those +whose "radial" distance is less than or equal to the given radius. The "radial +distance" (also called the "king's move" distance by some) looks like this: + +```text +3333333 +3222223 +3211123 +3210123 +3211123 +3222223 +3333333 +``` + +And the function: + +```clojure +(defn radial-distance + "Return the radial distance between two points." + [[x1 y1] [x2 y2]] + (max (abs (- x1 x2)) + (abs (- y1 y2)))) +``` + +I may end up needing to modify this `send-message-nearby` function to be a bit +more powerful in the future. Trystan's version modifies the verbs and such. +For now this is good enough for me. + +Now I can make lichens notify nearby creatures when they grow: + +```clojure +(defn grow [{:keys [location]} world] + (if-let [target (find-empty-neighbor world location)] + (let [new-lichen (make-lichen target) + world (assoc-in world [:entities (:id new-lichen)] new-lichen) + world (send-message-nearby location "The lichen grows." world)] + world) + world)) +``` + +The last step is to actually display these messages to the player. I added +a `draw-messages` function to the `ui/drawing.clj` file: + +```clojure +(defn draw-messages [screen messages] + (doseq [[i msg] (enumerate messages)] + (s/put-string screen 0 i msg {:fg :black :bg :white}))) +``` + +And then I modified the main `draw-ui` function for `:play` UIs to draw the +messages on top of the map: + +```clojure +(defmethod draw-ui :play [ui game screen] + (let [world (:world game) + {:keys [tiles entities]} world + player (:player entities) + [cols rows] (s/get-size screen) + vcols cols + vrows (dec rows) + origin (get-viewport-coords game (:location player) vcols vrows)] + (draw-world screen vrows vcols origin tiles) + (doseq [entity (vals entities)] + (draw-entity screen origin vrows vcols entity)) + (draw-hud screen game) + (draw-messages screen (:messages player)) + (highlight-player screen origin player))) +``` + +This does mean that messages will cover a bit of the screen. For now I'll live +with that, but in the future it's something to fix. + +Finally we need to clear the message queue out periodically, otherwise it'll +grow until it covers the entire screen! I modified the main game loop in +`core.clj`: + +```clojure +(defn clear-messages [game] + (assoc-in game [:world :entities :player :messages] nil)) + +(defn run-game [game screen] + (loop [{:keys [input uis] :as game} game] + (when (seq uis) + (if (nil? input) + (let [game (update-in game [:world] tick-all) + _ (draw-game game screen) + game (clear-messages game)] + (recur (get-input game screen))) + (recur (process-input (dissoc game :input) input)))))) +``` + +Results +------- + +After fixing a few other bugs (you can read the changelog if you're interested) +I've now got a working combat system, and a messaging system so I can tell +what's going on: + +![Screenshot](/media/images/blog/2012/07/caves-06-01.png) + +It's actually starting to feel like a real game now, instead of just a sandbox +where you can break things. + +You can view the code [on GitHub][result-code] if you want to see the end +result. + +[result-code]: https://github.com/sjl/caves/tree/entry-06/src/caves + +The next article will move on to Trystan's seventh post, which adds multiple +z-levels to the caves. + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/07/caves-of-clojure-interlude-1.html --- a/content/blog/2012/07/caves-of-clojure-interlude-1.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,559 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "The Caves of Clojure: Interlude 1" - snip: "Black magic." - created: 2012-07-14 17:06:00 - %} - -{% block article %} - -This post is part of an ongoing series. If you haven't already done so, you -should probably start at [the beginning][]. - -This entry is an interlude after [post five in Trystan's tutorial][trystan-tut]. - -If you want to follow along, the code for the series is [on Bitbucket][bb] and -[on GitHub][gh]. Update to the `interlude-1` tag to see the code as it stands -after this post. - -[the beginning]: /blog/2012/07/caves-of-clojure-01/ -[trystan-tut]: http://trystans.blogspot.com/2011/09/roguelike-tutorial-05-stationary.html -[bb]: http://bitbucket.org/sjl/caves/ -[gh]: http://github.com/sjl/caves/ - -[TOC] - -Summary -------- - -At the end of the last post I said that this one would be about refactoring and -a macro, so that's what it's going to be. - -Refactoring ------------ - -I don't want to bore you with lots of long chunks of refactored code, so I'll -just outline the changes and if you want to see what happened you can look in -the repo on GitHub. - -I ran [Kibit][] over the code and fixed what it complained. - -[jdmarble][] on GitHub pointed out a place where I was using `update-in` and -could simplify it to `assoc-in`. This is the kind of thing Kibit should catch, -so I added it in and sent a pull request. - -[jmgimeno][] on GitHub pointed out that I could use an `atom` for the entity ID -generation instead of a `ref`. That cleaned up a few lines nicely. - -I updated the game loop and moved the call to `draw-game` so it doesn't get draw -more times than is necessary. - -I added a bunch of comments and docstrings throughout the code. - -I added a few functions to the latest release of [clojure-lanterna][] that -allowed me to clean up the UI drawing code. I was able to completely remove the -`clear-screen` function, and replaced the hardcoded screen size with a dymanic -lookup. - -I also changed how the actual screen gets drawn. Look in the repo for the full -details -- I think it's much nicer now (though I'm still not 100% happy). - -I think that's about it. On to the meat of this post. - -[kibit]: https://github.com/jonase/kibit/ -[jdmarble]: https://github.com/jdmarble -[jmgimeno]: https://github.com/jmgimeno -[clojure-lanterna]: http://sjl.bitbucket.org/clojure-lanterna/ - -The Problem ------------ - -Right now, the entity system in the Caves works like this: - -* Aspects are Clojure protocols. They define what functions an entity must - implement to have that aspect. -* Entity types use `extend-type` to add the appropriate implementations for the - aspects they want to be. - -This is all vanilla Clojure, and up until now it's been fine because there was -no crossover between the `Lichen` aspects and the `Player` aspects. But what's -going to happen when I create a creature that shares behavior with one or both -of them? - -To see the problem, I'm going to create a `Bunny` entity that will hop around -the screen, and can also be destroyed (assuming the player is a terrible, -terrible person). So I'll create `entities/bunny.clj`: - - :::clojure - (ns caves.entities.bunny) - - (defrecord Bunny [id glyph color location hp]) - - (extend-type Bunny Entity - (tick [this world] - world)) - -We'll worry about `tick` soon, but so far, so good. Now I need to let bunnies -move around: - - :::clojure - (extend-type Bunny Mobile - (move [this world dest] - {:pre [(can-move? this world dest)]} - (assoc-in world [:entities (:id this) :location] dest)) - (can-move? [this world dest] - (is-empty? world dest))) - -Hmm, where have we seen this code before? - -It's an almost exact copy of the `Player`'s implementation of the `Mobile` -protocol! - -When you think about it, this makes sense. Most of the entities in the game -will have the same implementation for most aspects. The flexibility of Clojure -protocols means I have the power to customize behavior for every one of them, -but it also means that I have to redefine the same behavior over and over. - -Or do I? - -The Not-Quite Solutions ------------------------ - -There are a number of ways I could try to get around this duplication. - -First, I could define the "default" implementations as separate, normal -functions, and then the entity-specific implementations could just call those. - -This would work, absolutely. It would isolate the generic functionality in one -place successfully. But it means I'd have to manually type out the calls to -those generic functions all the time. This is a Lisp -- I can do better. - -The next idea is to make `Object` implement every aspect (with the default -implementations). This isn't ideal for two reasons. - -First, it means that the type of an entity is no longer useful. If `Object` -implements `Mobile` to provide the default functionality, it means *every* -entity will effectively be `Mobile` even if it shouldn't be! - -Second, it doesn't even give me everything I want. Observe: - - :::clojure - (defprotocol Foo - (hello [this]) - (world [this])) - - (defrecord A []) - - (extend-type Object Foo - (hello [this] :hello-object) - (world [this] :world-object)) - - (extend-type A Foo - (hello [this] :hello-a)) - - (def a (->A)) - - (hello a) ; Works - (world a) ; Doesn't work - -In this example, the `Foo` object doesn't get the benefit of the default -implementation because it implements the protocol itself. So when figuring out -what `world` function to call Clojure asks "Hmm, does A implement Foo? Oh, it -does? Okay, I'll use A's implementations then". - -So the entities would either have to implement all of the aspect functions -(resulting in the duplication of the ones they don't need to change) or none of -them (which *would* give them the defaults). - -So this isn't ideal. I could also have used multimethods here, because they -*do* support default implementations. But multimethods don't give me a nice -way to group related functions together like protocols. - -Protocols also interact with the type system to give me handy functionality -like: - - :::clojure - (defn find-targets - "Find potential things to kill!" - [world] - (filter #(satisfies? Destructible %) - (vals (:entities world)))) - -The concept of "bunnies are destructible" is a useful one, and I'd lose it if -I used multimethods. - -The Macro ---------- - -Macros are not something you should reach for right away. They're tricky and -much harder to understand than a normal function. But when all else fails, -they're there as a last resort. - -I couldn't figure out a way to do this without macros, so it's time to roll up -my sleeves and work some dark Lispy magic to get a nice syntax. - -When I'm writing a macro, my first step is usually to start at the end by -writing out what I want its ultimate usage to be. For this functionality I'm -actually going to need a pair of macros. - -First, `defaspect` will replace `defprotocol` and allow me to define a protocol -*and* provide the default implementations: - - :::clojure - (defaspect Mobile - (move [this world dest] - {:pre [(can-move? this world dest)]} - (assoc-in world [:entities (:id this) :location] dest)) - (can-move? [this world dest] - (is-empty? world dest))) - -Those implementations are generic enough (moving moves an entity from their -current space into an empty one) that many entities will probably be able to use -them unchanged. - -I'll also need a macro to replace `extend-type`. I decided to call it -`add-aspect`: - - :::clojure - (add-aspect EarthElemental Mobile - (can-move? [this world] - (entity-at? world dest))) - -In this example the `EarthElemental` entity is implementing `Mobile`. It will -use the default `move` implementation (which just changes its location), but it -overrides `can-move?`. Earth elementals can't walk through other entities, but -they *can* walk through the rock walls of the Caves. - -So I've got my examples of usage, now it's time to implement the macros. I'll -start with `defaspect`. - -My second step when writing a macro is writing out what the usage should be -expanded into. After a bit of thinking I came up with this: - - :::clojure - (defaspect Mobile - (move [this world dest] - {:pre [(can-move? this world dest)]} - (assoc-in world [:entities (:id this) :location] dest)) - (can-move? [this world dest] - (is-empty? world dest))) - - ; should expand into - - (do - (defprotocol Mobile - (move [this world dest]) - (can-move? [this world dest])) - (def Mobile - (with-meta Mobile - {:defaults - {:move (fn [this world dest] - {:pre [(can-move? this world dest)]} - (assoc-in world [:entities (:id this) :location] dest)) - :can-move? (fn [this world dest] - (is-empty? world dest))}}))) - -This looks a bit complicated because of the method implementations, which aren't -really important when writing the macro, so let's remove those: - - :::clojure - (defaspect Mobile - (move [this world dest] - {:pre [(can-move? this world dest)]} - (assoc-in world [:entities (:id this) :location] dest)) - (can-move? [this world dest] - (is-empty? world dest))) - - ; should expand into - - (do - (defprotocol Mobile - (move [this world dest]) - (can-move? [this world dest])) - (def Mobile - (with-meta Mobile - {:defaults - {:move (fn [this world dest] ...) - :can-move? (fn [this world dest] ...)}}))) - -That's a bit easier to read. The `defaspect` macro is going to take all the -forms I give it and expand into a `do` form with two actions: defining the -protocol as before, and attaching a map to the Protocol itself with Clojure's -metadata feature. - -This map will contain the default implementations. For now just trust me that -I'm going to need them in a map later. - -Now to write the actual macro! It'll go in `entities/core.clj` for the moment. -I'll start with a skeleton: - - :::clojure - (defmacro defaspect [label & fns] - (let [fnmap (make-fnmap fns) - fnheads (make-fnheads fns)] - `(do - (defprotocol ~label - ~@fnheads) - (def ~label - (with-meta ~label {:defaults ~fnmap}))))) - -If you've used macros before, this should be pretty easy to read. I've pulled -as much functionality as possible into two helper functions. Let's look at -those: - - :::clojure - (defn make-fnmap - "Make a function map out of the given sequence of fnspecs. - - A function map is a map of functions that you'd pass to extend. For example, - this sequence of fnspecs: - - ((foo [a] (println a) - (bar [a b] (+ a b))) - - Would be turned into this fnmap: - - {:foo (fn [a] (println a)) - :bar (fn [a b] (+ a b))} - - " - [fns] - (into {} (for [[label fntail] (map (juxt first rest) fns)] - [(keyword label) - `(fn ~@fntail)]))) - - (defn make-fnheads - "Make a sequence of fnheads of of the given sequence of fnspecs. - - A fnhead is a sequence of (name args) like you'd pass to defprotocol. For - example, this sequence of fnspecs: - - ((foo [a] (println a)) - (bar [a b] (+ a b))) - - Would be turned into this sequence of fnheads: - - ((foo [a]) - (bar [a b])) - - " - [fns] - (map #(take 2 %) fns)) - -Hopefully the docstrings will make them pretty clear. If you have questions let -me know (or play around with them in a REPL to see how they behave). - -And with that, `defaspect` is complete! I now have a way to define a protocol -and attach some default implementations to it in one easy, beautiful call. - -The other macro, `add-aspect`, is a piece of cake now that I've got the helper -functions: - - :::clojure - (defmacro add-aspect [entity aspect & fns] - (let [fnmap (make-fnmap fns)] - `(extend ~entity ~aspect (merge (:defaults (meta ~aspect)) - ~fnmap)))) - -The important thing in understanding this macro is `extend`. `extend` is what -`extend-type` and `extend-protocol` sugar over. It takes a type, a protocol, -and a map of the implementations of that protocol's functions. - -The key word there is "map", which really does mean a plain old Clojure map. So -this macro will expand like so: - - :::clojure - (add-aspect EarthElemental Mobile - (can-move? [this world dest] - (entity-at? world dest))) - - ; should expand into - - (extend EarthElemental Mobile - (merge (:defaults (meta Mobile)) - {:can-move? (fn [this world dest] ...)}) - -The `(:defaults (meta Mobile))` simply retrieves the function mapping that -`defaspect` attached to the Protocol, so in effect I get something like: - - :::clojure - (extend EarthElemental Mobile - (merge {:move (fn [this world dest] ...) - :can-move? (fn [this world dest] ...)} - {:can-move? (fn [this world dest] ...)}) - -`merge` is just the vanilla Clojure `merge` function, so the resulting map will -have the default implementations overridden by any custom ones given. - -And that's it! Let's see it in action. - -Usage ------ - -First I need to update the aspects to use `defaspect` and include their default -implementations. Here's `Destructible`: - - :::clojure - (ns caves.entities.aspects.destructible - (:use [caves.entities.core :only [defaspect]])) - - - (defaspect Destructible - (take-damage [{:keys [id] :as this} world damage] - (let [damaged-this (update-in this [:hp] - damage)] - (if-not (pos? (:hp damaged-this)) - (update-in world [:entities] dissoc id) - (assoc-in world [:entities id] damaged-this))))) - -The code is just torn out of the `Lichen` code. Since this is how lichens will -act, I can update them to use `add-aspect`: - - :::clojure - (add-aspect Lichen Destructible) - -One line! Nice. - -I'll make bunnies destructible too: - - :::clojure - (add-aspect Bunny Destructible) - -Perfect. I then updated `Mobile` to use the `defaspect` macro. Look in the -repository if you want to see that. Now players and bunnies can both use the -same default implementations for movement: - - :::clojure - (add-aspect Bunny Mobile) - (add-aspect Player Mobile) - -Beautiful. I then converted the remaining aspects and implementations to use -these macros. - -Let's add some bunnies to the world. First I'll need a `make-bunny` function -similar to `make-lichen`: - - :::clojure - (defn make-bunny [location] - (->Bunny (get-id) "v" :yellow location 1)) - -I don't know why I picked yellow. Are there yellow bunnies? There are in -*this* world. I used a `v` as the glyph because it kind of looks like bunny -ears. - -Then I updated the world-populating code over in `input.clj`: - - :::clojure - (defn add-creature [world make-creature] - (let [creature (make-creature (find-empty-tile world))] - (assoc-in world [:entities (:id creature)] creature))) - - (defn add-creatures [world make-creature n] - (nth (iterate #(add-creature % make-creature) - world) - n)) - - (defn populate-world [world] - (let [world (assoc-in world [:entities :player] - (make-player (find-empty-tile world)))] - (-> world - (add-creatures make-lichen 30) - (add-creatures make-bunny 20)))) - -Bunnies will be a bit rarer than lichens for the moment (and maybe in the future -they could eat them). - -Finally, let's run the game! - -![Screenshot](/media/images{{ parent_url }}/caves-interlude-1-01.png) - -Bunnies! They're populated into the world and the player can kill them because -they're `Destructible`. - -Right now they *can* move, but choose not to. I'll fix that by updating their -`tick` function: - - :::clojure - (extend-type Bunny Entity - (tick [this world] - (if-let [target (find-empty-neighbor world (:location this))] - (move this world target) - world))) - -Now they'll move whenever they're not backed into a corner. Obviously move -complicated AI is possible, but this is fine for now. - -So this is all good, but I haven't even used the "override one function of an -aspect but not others" part of my fancy macro. I'll add another creature to -show that off: - - - :::clojure - (ns caves.entities.silverfish - (:use [caves.entities.core :only [Entity get-id add-aspect]] - [caves.entities.aspects.destructible :only [Destructible]] - [caves.entities.aspects.mobile :only [Mobile move can-move?]] - [caves.world :only [get-entity-at]] - [caves.coords :only [neighbors]])) - - - (defrecord Silverfish [id glyph color location hp]) - - (defn make-silverfish [location] - (->Silverfish (get-id) "~" :white location 1)) - - - (extend-type Silverfish Entity - (tick [this world] - (let [target (rand-nth (neighbors (:location this)))] - (if (get-entity-at world target) - world - (move this world target))))) - - (add-aspect Silverfish Mobile - (can-move? [this world dest] - (not (get-entity-at world dest)))) - - (add-aspect Silverfish Destructible) - - -Oh no! The horrible silverfish from Minecraft! They wormy little guys are -`Mobile` and `Destructible`, but they can move through walls with their custom -`can-move?` function. - -Notice how I didn't have to provide an implementation of `move`. Silverfish -move like any other mobile entity (just by updating their location), the only -thing special is where they can go. - -After adding them to the world population, we can see a few wriggling their way -though the walls in the northeast corner: - -![Screenshot](/media/images{{ parent_url }}/caves-interlude-1-02.png) - -Results -------- - -You can view the code [on GitHub][result-code] if you want to see the end -result. - -[result-code]: https://github.com/sjl/caves/tree/interlude-1/src/caves - -These two macros allowed me to add a new mob, with custom movement, in about 13 -lines of code (excluding imports). That's pretty nice! They certainly aren't -perfect though. - -For one, every aspect *has* to define default implementations for its methods. -You can't force all entities to implement it. This isn't a big deal for now, -but I may want to add it in later. - -Second, it doesn't handle docstrings properly. Again, not a huge problem at the -moment but something to put on the todo list for later. - -Finally, defining the protocol and implementations in the same form may lead to -some tricky circular import issues. I can always split them into separate files -in the future though. - -That about wraps it up. If you have any questions or comments let me know! In -the next entry I'll get back to Trystan's tutorial. - -{% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/07/caves-of-clojure-interlude-1.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2012/07/caves-of-clojure-interlude-1.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,577 @@ ++++ +title = "The Caves of Clojure: Interlude 1" +snip = "Black magic." +date = 2012-07-14T17:06:00Z +draft = false + ++++ + +This post is part of an ongoing series. If you haven't already done so, you +should probably start at [the beginning][]. + +This entry is an interlude after [post five in Trystan's tutorial][trystan-tut]. + +If you want to follow along, the code for the series is [on Bitbucket][bb] and +[on GitHub][gh]. Update to the `interlude-1` tag to see the code as it stands +after this post. + +[the beginning]: /blog/2012/07/caves-of-clojure-01/ +[trystan-tut]: http://trystans.blogspot.com/2011/09/roguelike-tutorial-05-stationary.html +[bb]: http://bitbucket.org/sjl/caves/ +[gh]: http://github.com/sjl/caves/ + +{{% toc %}} + +Summary +------- + +At the end of the last post I said that this one would be about refactoring and +a macro, so that's what it's going to be. + +Refactoring +----------- + +I don't want to bore you with lots of long chunks of refactored code, so I'll +just outline the changes and if you want to see what happened you can look in +the repo on GitHub. + +I ran [Kibit][] over the code and fixed what it complained. + +[jdmarble][] on GitHub pointed out a place where I was using `update-in` and +could simplify it to `assoc-in`. This is the kind of thing Kibit should catch, +so I added it in and sent a pull request. + +[jmgimeno][] on GitHub pointed out that I could use an `atom` for the entity ID +generation instead of a `ref`. That cleaned up a few lines nicely. + +I updated the game loop and moved the call to `draw-game` so it doesn't get draw +more times than is necessary. + +I added a bunch of comments and docstrings throughout the code. + +I added a few functions to the latest release of [clojure-lanterna][] that +allowed me to clean up the UI drawing code. I was able to completely remove the +`clear-screen` function, and replaced the hardcoded screen size with a dymanic +lookup. + +I also changed how the actual screen gets drawn. Look in the repo for the full +details — I think it's much nicer now (though I'm still not 100% happy). + +I think that's about it. On to the meat of this post. + +[kibit]: https://github.com/jonase/kibit/ +[jdmarble]: https://github.com/jdmarble +[jmgimeno]: https://github.com/jmgimeno +[clojure-lanterna]: http://sjl.bitbucket.org/clojure-lanterna/ + +The Problem +----------- + +Right now, the entity system in the Caves works like this: + +* Aspects are Clojure protocols. They define what functions an entity must + implement to have that aspect. +* Entity types use `extend-type` to add the appropriate implementations for the + aspects they want to be. + +This is all vanilla Clojure, and up until now it's been fine because there was +no crossover between the `Lichen` aspects and the `Player` aspects. But what's +going to happen when I create a creature that shares behavior with one or both +of them? + +To see the problem, I'm going to create a `Bunny` entity that will hop around +the screen, and can also be destroyed (assuming the player is a terrible, +terrible person). So I'll create `entities/bunny.clj`: + +```clojure +(ns caves.entities.bunny) + +(defrecord Bunny [id glyph color location hp]) + +(extend-type Bunny Entity + (tick [this world] + world)) +``` + +We'll worry about `tick` soon, but so far, so good. Now I need to let bunnies +move around: + +```clojure +(extend-type Bunny Mobile + (move [this world dest] + {:pre [(can-move? this world dest)]} + (assoc-in world [:entities (:id this) :location] dest)) + (can-move? [this world dest] + (is-empty? world dest))) +``` + +Hmm, where have we seen this code before? + +It's an almost exact copy of the `Player`'s implementation of the `Mobile` +protocol! + +When you think about it, this makes sense. Most of the entities in the game +will have the same implementation for most aspects. The flexibility of Clojure +protocols means I have the power to customize behavior for every one of them, +but it also means that I have to redefine the same behavior over and over. + +Or do I? + +The Not-Quite Solutions +----------------------- + +There are a number of ways I could try to get around this duplication. + +First, I could define the "default" implementations as separate, normal +functions, and then the entity-specific implementations could just call those. + +This would work, absolutely. It would isolate the generic functionality in one +place successfully. But it means I'd have to manually type out the calls to +those generic functions all the time. This is a Lisp — I can do better. + +The next idea is to make `Object` implement every aspect (with the default +implementations). This isn't ideal for two reasons. + +First, it means that the type of an entity is no longer useful. If `Object` +implements `Mobile` to provide the default functionality, it means *every* +entity will effectively be `Mobile` even if it shouldn't be! + +Second, it doesn't even give me everything I want. Observe: + +```clojure +(defprotocol Foo + (hello [this]) + (world [this])) + +(defrecord A []) + +(extend-type Object Foo + (hello [this] :hello-object) + (world [this] :world-object)) + +(extend-type A Foo + (hello [this] :hello-a)) + +(def a (->A)) + +(hello a) ; Works +(world a) ; Doesn't work +``` + +In this example, the `Foo` object doesn't get the benefit of the default +implementation because it implements the protocol itself. So when figuring out +what `world` function to call Clojure asks "Hmm, does A implement Foo? Oh, it +does? Okay, I'll use A's implementations then". + +So the entities would either have to implement all of the aspect functions +(resulting in the duplication of the ones they don't need to change) or none of +them (which *would* give them the defaults). + +So this isn't ideal. I could also have used multimethods here, because they +*do* support default implementations. But multimethods don't give me a nice +way to group related functions together like protocols. + +Protocols also interact with the type system to give me handy functionality +like: + +```clojure +(defn find-targets + "Find potential things to kill!" + [world] + (filter #(satisfies? Destructible %) + (vals (:entities world)))) +``` + +The concept of "bunnies are destructible" is a useful one, and I'd lose it if +I used multimethods. + +The Macro +--------- + +Macros are not something you should reach for right away. They're tricky and +much harder to understand than a normal function. But when all else fails, +they're there as a last resort. + +I couldn't figure out a way to do this without macros, so it's time to roll up +my sleeves and work some dark Lispy magic to get a nice syntax. + +When I'm writing a macro, my first step is usually to start at the end by +writing out what I want its ultimate usage to be. For this functionality I'm +actually going to need a pair of macros. + +First, `defaspect` will replace `defprotocol` and allow me to define a protocol +*and* provide the default implementations: + +```clojure +(defaspect Mobile + (move [this world dest] + {:pre [(can-move? this world dest)]} + (assoc-in world [:entities (:id this) :location] dest)) + (can-move? [this world dest] + (is-empty? world dest))) +``` + +Those implementations are generic enough (moving moves an entity from their +current space into an empty one) that many entities will probably be able to use +them unchanged. + +I'll also need a macro to replace `extend-type`. I decided to call it +`add-aspect`: + +```clojure +(add-aspect EarthElemental Mobile + (can-move? [this world] + (entity-at? world dest))) +``` + +In this example the `EarthElemental` entity is implementing `Mobile`. It will +use the default `move` implementation (which just changes its location), but it +overrides `can-move?`. Earth elementals can't walk through other entities, but +they *can* walk through the rock walls of the Caves. + +So I've got my examples of usage, now it's time to implement the macros. I'll +start with `defaspect`. + +My second step when writing a macro is writing out what the usage should be +expanded into. After a bit of thinking I came up with this: + +```clojure +(defaspect Mobile + (move [this world dest] + {:pre [(can-move? this world dest)]} + (assoc-in world [:entities (:id this) :location] dest)) + (can-move? [this world dest] + (is-empty? world dest))) + +; should expand into + +(do + (defprotocol Mobile + (move [this world dest]) + (can-move? [this world dest])) + (def Mobile + (with-meta Mobile + {:defaults + {:move (fn [this world dest] + {:pre [(can-move? this world dest)]} + (assoc-in world [:entities (:id this) :location] dest)) + :can-move? (fn [this world dest] + (is-empty? world dest))}}))) +``` + +This looks a bit complicated because of the method implementations, which aren't +really important when writing the macro, so let's remove those: + +```clojure +(defaspect Mobile + (move [this world dest] + {:pre [(can-move? this world dest)]} + (assoc-in world [:entities (:id this) :location] dest)) + (can-move? [this world dest] + (is-empty? world dest))) + +; should expand into + +(do + (defprotocol Mobile + (move [this world dest]) + (can-move? [this world dest])) + (def Mobile + (with-meta Mobile + {:defaults + {:move (fn [this world dest] ...) + :can-move? (fn [this world dest] ...)}}))) +``` + +That's a bit easier to read. The `defaspect` macro is going to take all the +forms I give it and expand into a `do` form with two actions: defining the +protocol as before, and attaching a map to the Protocol itself with Clojure's +metadata feature. + +This map will contain the default implementations. For now just trust me that +I'm going to need them in a map later. + +Now to write the actual macro! It'll go in `entities/core.clj` for the moment. +I'll start with a skeleton: + +```clojure +(defmacro defaspect [label & fns] + (let [fnmap (make-fnmap fns) + fnheads (make-fnheads fns)] + `(do + (defprotocol ~label + ~@fnheads) + (def ~label + (with-meta ~label {:defaults ~fnmap}))))) +``` + +If you've used macros before, this should be pretty easy to read. I've pulled +as much functionality as possible into two helper functions. Let's look at +those: + +```clojure +(defn make-fnmap + "Make a function map out of the given sequence of fnspecs. + + A function map is a map of functions that you'd pass to extend. For example, + this sequence of fnspecs: + + ((foo [a] (println a) + (bar [a b] (+ a b))) + + Would be turned into this fnmap: + + {:foo (fn [a] (println a)) + :bar (fn [a b] (+ a b))} + + " + [fns] + (into {} (for [[label fntail] (map (juxt first rest) fns)] + [(keyword label) + `(fn ~@fntail)]))) + +(defn make-fnheads + "Make a sequence of fnheads of of the given sequence of fnspecs. + + A fnhead is a sequence of (name args) like you'd pass to defprotocol. For + example, this sequence of fnspecs: + + ((foo [a] (println a)) + (bar [a b] (+ a b))) + + Would be turned into this sequence of fnheads: + + ((foo [a]) + (bar [a b])) + + " + [fns] + (map #(take 2 %) fns)) +``` + +Hopefully the docstrings will make them pretty clear. If you have questions let +me know (or play around with them in a REPL to see how they behave). + +And with that, `defaspect` is complete! I now have a way to define a protocol +and attach some default implementations to it in one easy, beautiful call. + +The other macro, `add-aspect`, is a piece of cake now that I've got the helper +functions: + +```clojure +(defmacro add-aspect [entity aspect & fns] + (let [fnmap (make-fnmap fns)] + `(extend ~entity ~aspect (merge (:defaults (meta ~aspect)) + ~fnmap)))) +``` + +The important thing in understanding this macro is `extend`. `extend` is what +`extend-type` and `extend-protocol` sugar over. It takes a type, a protocol, +and a map of the implementations of that protocol's functions. + +The key word there is "map", which really does mean a plain old Clojure map. So +this macro will expand like so: + +```clojure +(add-aspect EarthElemental Mobile + (can-move? [this world dest] + (entity-at? world dest))) + +; should expand into + +(extend EarthElemental Mobile + (merge (:defaults (meta Mobile)) + {:can-move? (fn [this world dest] ...)}) +``` + +The `(:defaults (meta Mobile))` simply retrieves the function mapping that +`defaspect` attached to the Protocol, so in effect I get something like: + +```clojure +(extend EarthElemental Mobile + (merge {:move (fn [this world dest] ...) + :can-move? (fn [this world dest] ...)} + {:can-move? (fn [this world dest] ...)}) +``` + +`merge` is just the vanilla Clojure `merge` function, so the resulting map will +have the default implementations overridden by any custom ones given. + +And that's it! Let's see it in action. + +Usage +----- + +First I need to update the aspects to use `defaspect` and include their default +implementations. Here's `Destructible`: + +```clojure +(ns caves.entities.aspects.destructible + (:use [caves.entities.core :only [defaspect]])) + + +(defaspect Destructible + (take-damage [{:keys [id] :as this} world damage] + (let [damaged-this (update-in this [:hp] - damage)] + (if-not (pos? (:hp damaged-this)) + (update-in world [:entities] dissoc id) + (assoc-in world [:entities id] damaged-this))))) +``` + +The code is just torn out of the `Lichen` code. Since this is how lichens will +act, I can update them to use `add-aspect`: + +```clojure +(add-aspect Lichen Destructible) +``` + +One line! Nice. + +I'll make bunnies destructible too: + +```clojure +(add-aspect Bunny Destructible) +``` + +Perfect. I then updated `Mobile` to use the `defaspect` macro. Look in the +repository if you want to see that. Now players and bunnies can both use the +same default implementations for movement: + +```clojure +(add-aspect Bunny Mobile) +(add-aspect Player Mobile) +``` + +Beautiful. I then converted the remaining aspects and implementations to use +these macros. + +Let's add some bunnies to the world. First I'll need a `make-bunny` function +similar to `make-lichen`: + +```clojure +(defn make-bunny [location] + (->Bunny (get-id) "v" :yellow location 1)) +``` + +I don't know why I picked yellow. Are there yellow bunnies? There are in +*this* world. I used a `v` as the glyph because it kind of looks like bunny +ears. + +Then I updated the world-populating code over in `input.clj`: + +```clojure +(defn add-creature [world make-creature] + (let [creature (make-creature (find-empty-tile world))] + (assoc-in world [:entities (:id creature)] creature))) + +(defn add-creatures [world make-creature n] + (nth (iterate #(add-creature % make-creature) + world) + n)) + +(defn populate-world [world] + (let [world (assoc-in world [:entities :player] + (make-player (find-empty-tile world)))] + (-> world + (add-creatures make-lichen 30) + (add-creatures make-bunny 20)))) +``` + +Bunnies will be a bit rarer than lichens for the moment (and maybe in the future +they could eat them). + +Finally, let's run the game! + +![Screenshot](/media/images/blog/2012/07/caves-interlude-1-01.png) + +Bunnies! They're populated into the world and the player can kill them because +they're `Destructible`. + +Right now they *can* move, but choose not to. I'll fix that by updating their +`tick` function: + +```clojure +(extend-type Bunny Entity + (tick [this world] + (if-let [target (find-empty-neighbor world (:location this))] + (move this world target) + world))) +``` + +Now they'll move whenever they're not backed into a corner. Obviously move +complicated AI is possible, but this is fine for now. + +So this is all good, but I haven't even used the "override one function of an +aspect but not others" part of my fancy macro. I'll add another creature to +show that off: + + +```clojure +(ns caves.entities.silverfish + (:use [caves.entities.core :only [Entity get-id add-aspect]] + [caves.entities.aspects.destructible :only [Destructible]] + [caves.entities.aspects.mobile :only [Mobile move can-move?]] + [caves.world :only [get-entity-at]] + [caves.coords :only [neighbors]])) + + +(defrecord Silverfish [id glyph color location hp]) + +(defn make-silverfish [location] + (->Silverfish (get-id) "~" :white location 1)) + + +(extend-type Silverfish Entity + (tick [this world] + (let [target (rand-nth (neighbors (:location this)))] + (if (get-entity-at world target) + world + (move this world target))))) + +(add-aspect Silverfish Mobile + (can-move? [this world dest] + (not (get-entity-at world dest)))) + +(add-aspect Silverfish Destructible) + +``` + +Oh no! The horrible silverfish from Minecraft! They wormy little guys are +`Mobile` and `Destructible`, but they can move through walls with their custom +`can-move?` function. + +Notice how I didn't have to provide an implementation of `move`. Silverfish +move like any other mobile entity (just by updating their location), the only +thing special is where they can go. + +After adding them to the world population, we can see a few wriggling their way +though the walls in the northeast corner: + +![Screenshot](/media/images/blog/2012/07/caves-interlude-1-02.png) + +Results +------- + +You can view the code [on GitHub][result-code] if you want to see the end +result. + +[result-code]: https://github.com/sjl/caves/tree/interlude-1/src/caves + +These two macros allowed me to add a new mob, with custom movement, in about 13 +lines of code (excluding imports). That's pretty nice! They certainly aren't +perfect though. + +For one, every aspect *has* to define default implementations for its methods. +You can't force all entities to implement it. This isn't a big deal for now, +but I may want to add it in later. + +Second, it doesn't handle docstrings properly. Again, not a huge problem at the +moment but something to put on the todo list for later. + +Finally, defining the protocol and implementations in the same form may lead to +some tricky circular import issues. I can always split them into separate files +in the future though. + +That about wraps it up. If you have any questions or comments let me know! In +the next entry I'll get back to Trystan's tutorial. + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/10/a-modern-space-cadet.html --- a/content/blog/2012/10/a-modern-space-cadet.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1303 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "A Modern Space Cadet" - snip: "Emulating a legendary keyboard." - created: 2012-10-03 09:55:00 - %} - -{% block article %} - -I spend a lot of my time (easily over 8 hours a day) at a keyboard. As you -might have guessed from my previous entries about [Vim][] and [Mutt][] I'm not -averse to spending a few hours to improve an environment I'm going to be -spending tens of thousands of hours in over the next few years, so it shouldn't -be a shock that my keyboard is something I've heavily tweaked. - -This post is about what I've done to make my typing experience more pleasant and -efficient. - -If you scoff at customization you won't enjoy this post. What if I have to use -someone else's machine? I can count on one hand the number of times I've had to -use someone else's machine ever since I got an iPhone five years ago. I *like* -customizing my machine to save me time. Yes, there may be some excessive stuff -here, but not only does it make me type faster, it's *fun*! - -[TOC] - -[Vim]: /blog/2010/09/coming-home-to-vim/ -[Mutt]: /blog/2012/10/the-homely-mutt/ - -The Original ------------- - -There have been many, many keyboards produced in the world since the first ones -emerged. One of the most famous, at least in programming circles, is the [Space -Cadet Keyboard][space-cadet]. - -Originally used on Lisp machines and some other systems, it's not a keyboard for -the minimalist. There are four "bucky keys" (modifier keys): control, meta, -super, and hyper (plus shift, of course). There are also special keys that let -you type Greek letters and mathematical symbols. - -The Space Cadet is an example of a keyboard for someone not afraid to invest -some time to work faster. - -It's also absolutely gorgeous. The color scheme and typography is beautiful. - -This is the keyboard I used (loosely) as inspiration when crafting my current -setup. - -[space-cadet]: http://world.std.com/~jdostale/kbd/SpaceCadet.html - -Modern Hardware ---------------- - -I've tried a number of modern keyboards in the past few years. They're all -high-quality and more expensive than the $20 plastic toys that come with -desktops. But I spend 60 or more hours a week at a keyboard and maybe one hour -a week in my car, so I'm getting pretty good use out of the dollars I've put -into keyboards if you compare them to the cost of my car! - -I'll go through the keyboards I've used in chronological order. I'm not going -to write too much about the basics of mechanical keyboards and switches. If you -want to learn about that, [this post][ch] and [this guide][mech] are good places -to start. - -[ch]: http://www.codinghorror.com/blog/2010/10/the-keyboard-cult.html -[mech]: http://www.overclock.net/t/491752/mechanical-keyboard-guide - -### Apple Wireless Keyboard - -For a long time I used [Apple wireless keyboards][apple-wireless]. - -![Apple Wireless Keyboard](/media/images{{ parent_url }}/kb-apple.jpg) - -They're light and compact, but still feel extremely solid thanks to their metal -construction. - -They also stay really clean because there gaps between keys are tiny, instead of -the funnel-shaped gaps of other keyboard that send dirt straight to the bottom. - -They're readily available at any Apple store, and of course they're wireless -which is great. - -They also feel exactly like the keyboards on Apple's laptops, so your muscle -memory is perfectly suited to either one if you switch between them often. - -Unfortunately typing on them is nowhere near as nice as the rest of the -keyboards in this list. Apple keyboards have (I think) only 2mm of travel, but -you have to bottom-out the keys to register the keypresses. - -[apple-wireless]: http://www.apple.com/keyboard/ - -### Das Silent Ultimate - -The first mechanical keyboard I got was the [Das Silent Ultimate][das-silent]. - -![Das Silent Keyboard](/media/images{{ parent_url }}/kb-das.jpg) - -(This photo is actually my Das Clicky since I don't have the Silent any more, -but they're exactly the same externally.) - -Compared to the Apple keyboards the switches on the Das Silent (Cherry Brown -switches) feel far softer. You're not smashing your fingers against metal on -every keypress. - -Overall the Das Silent isn't too bad. It's built like a tank so I have no doubt -it'd last forever. It also has the option of blank keys, which I prefer. - -I didn't stick with this keyboard for long. First, its glossy finish looks good -when it's in the box and terrible forever after that. It's way too easy to get -dirty. - -Something also just feels "off" about the key switches to me, especially after -I've tried the other boards in this list. They feel, for lack of a better term, -"mushy" to me. - -[das-silent]: http://www.daskeyboard.com/model-s-ultimate-silent/ - -### Das Clicky Ultimate - -The next keyboard I tried was the [Das Clicky Ultimate][das-clicky]. - -![Das Clicky Keyboard](/media/images{{ parent_url }}/kb-das.jpg) - -This is exactly the same as the Das Silent except for the switches, which are -the "clicky" Cherry Blues. These feel *way* better to me than the browns. The -mushiness I described for the Silent is completely gone. - -They are not kidding when they say it is loud. If you're typing a lot expect to -have a constant background noise of clicks. But to me the sound is soothing, -especially once you get into a rhythm of typing for extended periods. Your -coworkers might disagree, so tread carefully. - -The Das Clicky still has that godawful glossy finish, unfortunately. After -a while I decided to try out what I had heard people raving about: Topre -switches. - -[das-clicky]: http://www.daskeyboard.com/model-s-ultimate/ - -### Happy Hacking Keyboard Professional 2 - -The [Happy Hacking Keyboard][hhkb] was my next keyboard. It's much more -expensive than the Dases were, so it was a tough call, but I'm glad I got it. - -![Happy Hacking Keyboard](/media/images{{ parent_url }}/kb-hhkb.jpg) - -The HHKB uses Topre key switches, which to me feel like Cherry Browns done -right. There's no "click" like the Blues, but instead of the mush of the -Browns you get a satisfying "thunk" or "sshhhunk" on each press. It's a great -balance: you have a nice satisfying sound that's less annoying for other people. - -The finish on this keyboard is perfect. It is the most unassuming, elegant, -wonderful matte finish I've seen on any keyboard. The plastic is just textured -enough to feel solid under your fingers (not slippery like Macbook keys) but not -enough to feel like you're typing on sandpaper. - -If you're short on space, or want to carry your keyboard with you, the HHKB is -for you. It's *tiny*, but still feels solid and not flimsy at all. - -I have two main complaints about the HHKB. First, there's two spaces on the -bottom row where they could have put a modifier key but didn't. The space on -the right has the logo, which is fair enough, but the one on the left is simply -blank for no good reason. - -My other complaint is the idiotic default placement of the `fn` key and the -arrow keys. I don't mind having to use `fn` to get to the arrows as -a concession to a compact design, but why on earth would you place them so that -you have to chord on a *single hand* to hit them? - -This problem would be completely solved if the `fn` key were moved from its -current position (right of the right shift key) down to the empty lower-left -corner. Then you could use your left hand to hold `fn` while your right worked -the arrow keys, no single-hand contortion required. - -So after a while with the HHKB I got sick of the arrow key problem. I loved the -Topre switches though, so the next choice was pretty obvious. - -[hhkb]: http://elitekeyboards.com/products.php?sub=pfu_keyboards,hhkbpro2&pid=pdkb400bn - -### Topre Realforce 103 - -The Topre Realforce 103UB was my next keyboard, and is the one I'm -still using to this day. - -![Topre Realforce Keyboard](/media/images{{ parent_url }}/kb-realforce.jpg) - -I believe the model I bought is now discontinued, but it's been replaced by the -[104UB][topre] which is exactly the same except for an extra key on the right -side. If I could swap my current keyboard for one of those I would. In fact, -if anyone wants to buy my used 103UB for a bit of a discount I'd be totally -willing to sell it to get the 104UB. - -Anyway, this keyboard has a few disadvantages. First, unlike every other -keyboard here (except the wireless Apple ones) *it is not a USB hub*. This -isn't a huge deal (I have external USB hubs), but it did come as a big surprise. - -Second, it's big. Really big. If you don't have *plenty* of room on your desk -you might want to look at the [tenkeyless][topre-tenkeyless] varieties which -ditch the number pad section to save space. I have room and I love having -a number pad, so I got the behemoth-sized model. - -Also for some reason I can't fathom there's no way to get blank, black keycaps -for this keyboard. You can get blank Topre keycaps in lime green, pink, yellow, -and lots of other colors, but not a simple black! - -Now for the good parts. This thing is built like a tank. If you needed to -defend your home from an intruder, it would make an acceptable blunt weapon with -which to do so (and you'd probably be able to plug it right back in and start -typing again). - -The finish is the same as on the HHKB (gorgeous), but I have to take a point off -for the non-blank keycaps. I don't need to look at the keys while I type, so -why marr the luxurious finish with lettering? - -The Realforce also uses Topre switches, but it has another trick up its sleeve -that makes typing on it even nicer than the HHKB. The version I have is the -"variable-weighted" one, which means that the keys under your weaker fingers -take less force to press than the ones under your stronger fingers. This is -fantastic for long sessions of typing. Instead of my pinkies getting tired my -fingers seem to all tire out at the same rate now. Topre does make a "uniform" -version that doesn't have this trick, but I really like variable weight keys -myself. - -[topre]: http://elitekeyboards.com/products.php?sub=topre_keyboards,rf104&pid=xf11t0 -[topre-tenkeyless]: http://elitekeyboards.com/products.php?sub=topre_keyboards,rftenkeyless - -### Verdict - -Right now I'm using the Topre Realforce for my day to day work. I've fallen in -love with how the Topre switches feel (I prefer them over all the others except -*maybe* the Cherry Blues for the sound) and the Happy Hacking Keyboard is just -too cramped. If I were short on space the HHKB would be great, but I have -plenty of room on my desk, so why not make use of it? - -Here's a photo of all the keyboards (along with a 13" Macbook Air) so you can -see the differences in size. - -![Keyboard Size Comparison](/media/images{{ parent_url }}/kb-size.jpg) - -Modern Software ---------------- - -Now that I've talked about the hardware it's time for the software. I use OS -X exclusively, so everything I say here is OS X specific. I'm sure there are -Windows and Linux equivalents somewhere though. - -I use three software programs that, together, give me just about unlimited -flexibility in customizing how my keyboard works in OS X. - -### Keyboard Maestro - -[Keyboard Maestro][] is a utility for binding macros to keyboard shortcuts in OS -X. It may not have the best user interface, but it can do a *lot*, and once you -set up your shortcuts you never have to look at it again. It's $36. - -Right now I use it for application switching, with a few twists I'll cover later -on. - -I switched to [Keymando][] for a while. It was great being able to configure it -in a programming language with a plain text file I could easily work with in -version control, but compared to Keyboard Maestro it's far slower and far -buggier. I'm now back to only using Keyboard Maestro. - -[Keyboard Maestro]: http://www.keyboardmaestro.com/main/ -[Keymando]: http://keymando.com/ - -### PCKeyboardHack - -[PCKeyboardHack][] lets you map one key on your keyboard to another at a very -low level. For example, you can change just the right Option key to send `F19` -instead. I use this for one single key which you'll see in the next section. -It's free and [open source][pc-git]. - -[PCKeyboardHack]: http://pqrs.org/macosx/keyremap4macbook/pckeyboardhack.html.en -[pc-git]: https://github.com/tekezo/PCKeyboardHack - -### KeyRemap4MacBook - -[KeyRemap4MacBook][] is how I do the bulk of my keyboard customizations. Like -all of the other applications its user interface is horrible, but it can do damn -near *anything* you might want. You could probably replace Keyboard Maestro -with it, at least for the kind of stuff I do. It's also free and [open -source][kr-git]. - -The bulk of what I describe in this post is going to use KeyRemap4MacBook. I'm -not going to give you a tutorial in it here -- read its documentation if you -want to learn how to use the things I'm going to show. - -[KeyRemap4MacBook]: http://pqrs.org/macosx/keyremap4macbook/index.html.en -[kr-git]: https://github.com/tekezo/KeyRemap4MacBook - -Control/Escape --------------- - -These first few mappings aren't directly from the Space Cadet keyboard, but they -were inspired by its spirit of making an efficient tool for text editing. - -The Capslock key on modern keyboard has become the punchline of many a joke, and -for good reason: there's no reason to dedicate such an important key to -a function like capslock. Many people rebind it to a more useful key like -Control, Option, Escape, or Backspace. I rebound it to Control for a while and -then realized that with KeyRemap4MacBook I could get even more mileage out of -it. - -The important thing I thought of one day is that it's possible to divide keys on -the keyboard into three groups: - -* Keys you hold down to change how *other* keys behave, but that (usually) don't - do anything if you use them on their own (like Shift and Control). -* Keys that you press and release but don't want to "repeat" as you hold them - (like Escape or Insert). -* Keys that you sometimes press and release, but sometimes want to repeat (like - holding Space to insert a bunch of spaces, or Backspace to kill a bunch of - characters). - -Can you see where this is heading? The last group is pretty normal, but the -first two groups are special. Specifically: there are two different ways to use -them and they're each only useful in one of those ways. - -This means that we can *combine* them onto a single key without losing any -useful functionality! - -I'm clearly not the first one to think of this, because KR4MB includes built-in -support for creating these kinds of mappings. - -First I've mapped Capslock to Control at the OS X level by going into the -Keyboard preference pane in System Preferences, clicking the Modifier Keys -button in the lower right, selecting the keyboard in the dropdown list (this is -surprisingly easy to miss), and changing the Capslock setting: - -![Changing Capslock to Control in OS X](/media/images{{ parent_url }}/kb-caps.png) - -Then I've selected the following premade option in KeyRemap4MacBook: - -> Control\_L to Control\_L -> (+ when you type Control\_L only, send Escape) - -Now the Capslock key on the keyboard does the following: - -1. If held down and pressed with another key, it acts like Control. -2. If pressed and released on its own, it acts like Escape. - -That's two extremely important keys (at least for Vim users) on a single key in -one of the prime locations on the keyboard! No more awkward stretches! - -If you try this and want to force yourself to learn to use it, disable the -normal escape key. You'll learn fast. - -**Update:** I forgot to mention that if you use this (and the Shift -improvements below) you'll probably want to change the "\[Key Overlaid -Modifier\] Timeout" setting in the KR4MB preferences: - -![Key Overlaid Timeout](/media/images{{ parent_url }}/kb-key-overlaid.png) - -This setting controls the maximum length of a keypress that will register as -"single press". It's easier to understand with an example. The default is -1000ms, which means that if you press and hold the Capslock key for 900ms and -release it, it will count as Escape. If you hold it for 1001ms before -releasing, it counts as pressing and releasing Control instead. - -If you're finding that you're sometimes holding Control out of habit and then -releasing it, but it's still firing as an Escape, you may want to turn this -setting down to a lower value. I have mine set to 300ms. - -This is a lot more noticeable with the Shift mappings I'll describe shortly, so -keep it in the back of your mind if you try those out. - -Hyper ------ - -Modern OS X supports four "modifier" keys: Control, Option, Command, and Shift. -The Space Cadet keyboard had five: Control, Meta, Super, Hyper, and Shift. - -Shift and Control map to each other, and OS X's Option key is pretty much the -same as Meta (in fact I think Emacs users usually use Alt as Meta). -I arbitrarily decided that Command was the OS X equivalent of Super. That left -Hyper. - -Since OS X doesn't natively support a fifth modifier key I had to come up with -a way to fake it. - -First, I realized that because I use Capslock for Control I had no use for the -*real* left Control key. So the first step was to remap that to something else -distinct from the Capslock-version of Control. - -This required PCKeyboardHack. I don't think there's a way to do it in -KeyRemap4MacBook, because by the time KR4MB sees the keypress it can't tell if -it came from the Capslock key or the real Control key (due to how they were -changed in the previous section). PCKeyboardHack, however, *can*, so I remapped -left Control to keycode `80`: - -![Remapping Left Control in PCKeyboardHack](/media/images{{ parent_url }}/kb-pck.png) - -Keycode `80` is the `F19` key. My keyboard doesn't have an `F19` key so it -doesn't conflict with anything. Now I can simply remap `F19` in KR4MB just like -any other key. - -To create a "pseudo-Hyper" modifier, I remapped this key to be the equivalent of -holding down *all four other modifiers* by adding the following to my -`private.xml` KR4MB configuration file: - - :::xml - - Remap Left Control to Hyper - OS X doesn't have a Hyper. This maps Left Control to Control + Shift + Option + Command. - - space_cadet.left_control_to_hyper - - - --KeyToKey-- - KeyCode::F19, - - KeyCode::COMMAND_L, - ModifierFlag::OPTION_L | ModifierFlag::SHIFT_L | ModifierFlag::CONTROL_L - - - -As far as I can tell, no keyboard shortcuts in OS X or any apps use all four -modifier keys (how would you normally press them all, anyway?). But many -programs, like Keyboard Maestro, let you define your own shortcuts. So now I've -got an entire key as a "namespace" all to myself for my own shortcuts! - -This is really nice. I don't have to worry about "shadowing" existing shortcuts -anywhere and it's only a single modifier to press. - -Currently I use this "namespace" for application switching. Instead of using -the normal `Command-Tab` switcher, I have shortcuts for each individual app -I use frequently. For example, `Hyper-k` switches to Firefox, `Hyper-y` -switches to Twitter, and so on. This is better than `Command-Tab`ing because -I don't have to worry about how many times I need to press it. `Hyper-k` -*always* goes to Firefox no matter what, so I can easily burn that into my -muscle memory. - -There's one more little trick I use in Keyboard Maestro. It's specific to how -I work, but I'm sure some of you will still find it handy. - -I pretty much always keep two iTerm 2 windows open. The first contains -a [tmux][] session with one window split into two panes. One pane holds -[weechat][weechat-prog] for IRC, the other holds [Mutt][mutt-prog] for email. -I keep this on my laptop screen at all times while I do other things on my -external monitor. - -The second iTerm window is almost always fullscreened, and contains a tmux -session with whatever I'm working on. The number of windows and panes varies -wildly. - -What I wanted was a way to bind `Hyper-i` and `Hyper-m` to directly focus my IRC -and mail panes, and `Hyper-j` to directly focus the second, "general-purpose" -iTerm window. - -The solution came in two parts. First I configured tmux so that `prefix N` -would select the nth pane in the current window by adding the following to -`~/.tmux.conf`: - - :::text - bind 1 select-pane -t 1 - bind 2 select-pane -t 2 - bind 3 select-pane -t 3 - bind 4 select-pane -t 4 - bind 5 select-pane -t 5 - bind 6 select-pane -t 6 - bind 7 select-pane -t 7 - bind 8 select-pane -t 8 - bind 9 select-pane -t 9 - -My tmux prefix is `Control-f`, so now pressing `Control-f 1` will go to pane 1, -and so on. Then I configured Keyboard Maestro like so: - -![Keyboard Maestro IRC Config](/media/images{{ parent_url }}/kb-irc.png) - -This binds `Control-Shift-Option-Command-i` (which is just `Hyper-i`) to do the -following: - -1. Focus iTerm 2. -2. Send a `Command-Option-1` keystroke, which will focus the first iTerm - 2 window (I simply make sure I always open my mail/irc window first). -3. Send the `Control-f` keystroke. -3. Send the `1` keystroke, which together with the previous one tells tmux to - switch to pane 1. - -The mail shortcut is similar, of course, and the general-purpose terminal one is -even simpler. - -So now I've got a free modifier key that won't conflict with anything, and I've -got some very easy-to-type shortcuts I can burn into my fingers for switching -applications quickly. Awesome. - -[tmux]: http://tmux.sourceforge.net/ -[weechat-prog]: http://www.weechat.org/ -[mutt-prog]: http://www.mutt.org/ - -Better Shifting ---------------- - -The Shift keys are another of those keys that are only useful with other keys, -so it's only natural that they were also on the list of keys to optimize. -First, however, I took a detour to correct a bad habit of mine. - -### Shift Key Training Wheels - -I've been typing for most of my life, but I never really learned to do it -correctly. I can touch type, of course, but sometimes I use the wrong fingers -for certain keys. - -My most egregious offense was that I always used the left Shift key. Even when -typing `X` I'd hold the left Shift key with my left pinky and hit the `x` with -my left index finger, which pulled my hand off the home row and generally felt -awkward. - -One day I decided I was tired of doing this the wrong way and resolved to fix -myself. The easiest way to break a bad habit is to make it harder or impossible -to do, so I created a custom KeyRemap4MacBook setting that *disables* the -keypress when you use the incorrect Shift key with a letter. Here it is in -full: - - :::xml - - Use the correct shift keys. - private.correct_shift_keys - - --KeyToKey-- KeyCode::Q, ModifierFlag::SHIFT_L, KeyCode::VK_NONE - --KeyToKey-- KeyCode::W, ModifierFlag::SHIFT_L, KeyCode::VK_NONE - --KeyToKey-- KeyCode::E, ModifierFlag::SHIFT_L, KeyCode::VK_NONE - --KeyToKey-- KeyCode::R, ModifierFlag::SHIFT_L, KeyCode::VK_NONE - --KeyToKey-- KeyCode::T, ModifierFlag::SHIFT_L, KeyCode::VK_NONE - --KeyToKey-- KeyCode::A, ModifierFlag::SHIFT_L, KeyCode::VK_NONE - --KeyToKey-- KeyCode::S, ModifierFlag::SHIFT_L, KeyCode::VK_NONE - --KeyToKey-- KeyCode::D, ModifierFlag::SHIFT_L, KeyCode::VK_NONE - --KeyToKey-- KeyCode::F, ModifierFlag::SHIFT_L, KeyCode::VK_NONE - --KeyToKey-- KeyCode::G, ModifierFlag::SHIFT_L, KeyCode::VK_NONE - --KeyToKey-- KeyCode::Z, ModifierFlag::SHIFT_L, KeyCode::VK_NONE - --KeyToKey-- KeyCode::X, ModifierFlag::SHIFT_L, KeyCode::VK_NONE - --KeyToKey-- KeyCode::C, ModifierFlag::SHIFT_L, KeyCode::VK_NONE - --KeyToKey-- KeyCode::V, ModifierFlag::SHIFT_L, KeyCode::VK_NONE - - --KeyToKey-- KeyCode::Y, ModifierFlag::SHIFT_R, KeyCode::VK_NONE - --KeyToKey-- KeyCode::U, ModifierFlag::SHIFT_R, KeyCode::VK_NONE - --KeyToKey-- KeyCode::I, ModifierFlag::SHIFT_R, KeyCode::VK_NONE - --KeyToKey-- KeyCode::O, ModifierFlag::SHIFT_R, KeyCode::VK_NONE - --KeyToKey-- KeyCode::P, ModifierFlag::SHIFT_R, KeyCode::VK_NONE - --KeyToKey-- KeyCode::H, ModifierFlag::SHIFT_R, KeyCode::VK_NONE - --KeyToKey-- KeyCode::J, ModifierFlag::SHIFT_R, KeyCode::VK_NONE - --KeyToKey-- KeyCode::K, ModifierFlag::SHIFT_R, KeyCode::VK_NONE - --KeyToKey-- KeyCode::L, ModifierFlag::SHIFT_R, KeyCode::VK_NONE - --KeyToKey-- KeyCode::SEMICOLON, ModifierFlag::SHIFT_R, KeyCode::VK_NONE - --KeyToKey-- KeyCode::N, ModifierFlag::SHIFT_R, KeyCode::VK_NONE - --KeyToKey-- KeyCode::M, ModifierFlag::SHIFT_R, KeyCode::VK_NONE - --KeyToKey-- KeyCode::COMMA, ModifierFlag::SHIFT_R, KeyCode::VK_NONE - --KeyToKey-- KeyCode::DOT, ModifierFlag::SHIFT_R, KeyCode::VK_NONE - --KeyToKey-- KeyCode::QUOTE, ModifierFlag::SHIFT_R, KeyCode::VK_NONE - - -It took me about four hours of normal computer use to unlearn over two decades -of wrong typing. It's amazing how fast muscle memory can adjust when you simply -*force* it to. - -Once I fixed myself I disabled this setting, because it also makes you to use -the "correct" Shift key when doing keyboard shortcuts, and sometimes those are -easer to do with the "wrong" Shift. - -If you only use one Shift key I'd really encourage you to try this. It spreads -out the wear over both pinkies and feels much better. Your hands will thank -you. - -### Shift Parentheses - -Now that I was using the right Shift keys, it was time to revisit mapping -something else onto them. One pair of keys stood out as a perfect candidate: the -left and right parentheses. - -Parentheses are common in most of the programming languages in use today (some -more than others). If you counted I bet you'd find them more common than square -or curly brackets in the majority of your code, and yet they're shoved away in -the horrible homes of `Shift-9` and `Shift-0`. - -I decided to try out remapping my Shift keys to work like so: - -* When held while pressing other keys, act like Shift. -* When pressed and released on their own, type an opening or closing parenthesis - (left and right shift respectively). - -This means I can type parentheses with a single, unchorded keypress. After -using it for a while I absolutely love it. Here's the KeyRemap4MacBook setting -for it: - - :::xml - - Shifts to Parentheses - Shifts, when pressed alone, type parentheses. When used with other keys they're normal shifts. - - private.shifts_to_parens - - - --KeyOverlaidModifier-- KeyCode::SHIFT_R, ModifierFlag::SHIFT_R | ModifierFlag::NONE, KeyCode::SHIFT_R, KeyCode::KEY_0, ModifierFlag::SHIFT_L - --KeyOverlaidModifier-- KeyCode::SHIFT_L, ModifierFlag::SHIFT_L | ModifierFlag::NONE, KeyCode::SHIFT_L, KeyCode::KEY_9, ModifierFlag::SHIFT_R - - - --KeyToKey-- KeyCode::SHIFT_L, ModifierFlag::SHIFT_R, KeyCode::KEY_0, ModifierFlag::SHIFT_L, KeyCode::KEY_9, ModifierFlag::SHIFT_L - --KeyToKey-- KeyCode::SHIFT_R, ModifierFlag::SHIFT_L, KeyCode::KEY_9, ModifierFlag::SHIFT_L, KeyCode::KEY_0, ModifierFlag::SHIFT_L - - - --KeyToKey-- KeyCode::SPACE, ModifierFlag::SHIFT_R, KeyCode::KEY_0, ModifierFlag::SHIFT_L, KeyCode::SPACE - - -As you can see there's actually a bit more to the mapping, because I was finding -some minor usability glitches in the bare setting. - -Occasionally I'll accidentally type a parenthesis when meaning to do something -else, but it happens so infrequently and the benefits are so great that I'd -absolutely recommend this to anyone who programs a lot. - -Key Layers ----------- - -Now we're getting to things that are directly inspired by the Space Cadet -keyboard. If you don't type a lot of mathematical characters you may not want -to bother with these, but since I've been taking the [Introduction to -Mathematical Thinking][coursera] class on Coursera these bindings have made it -very pleasant to type out my homework. - -[coursera]: https://www.coursera.org/course/maththink - -### Greek - -The Space Cadet had a separate key for typing Greek letters, and I've decided to -do the same. I don't know *exactly* how that modifier key worked, but here's -how my version works: - -* Press and release the "Greek" key to go into "Greek mode" for one character. -* Type the appropriate Latin letter to get the Greek character for that key - (capital letters and lowercase are distinct). - -For example, typing `Greek`, then `w` results in ω (omega), while `Greek`, then -`W` results in Ω (capital omega). - -If you want to actually type out full Greek words, this isn't ideal because you -have to press the `Greek` key between every letter. But for my purposes -(variables in math) it's perfect. - -There are two parts to implementing this in KeyRemap4MacBook. First I needed to -pick a `Greek` key. I never use the Command, Option, and Control keys to the -right of Space (another bad habit I should probably fix one day), so I chose the -right Option key for this. - -If you *do* use the modifiers on the right you might prefer to pick a different -key for this, like `Insert` or `Print Screen`. - -Also, since I'm using the Realforce which is a PC keyboard, that key is actually -called the `PC Application` key. I'm sure you can figure out how to adjust the -mapping if necessary. - -Here's the first part of the mapping, which binds the `Greek` key: - - :::xml - - Greek Layer - Right PC Application key activates the Greek key layer. - - space_cadet.greek_layer - - - --KeyToKey-- - KeyCode::PC_APPLICATION, - KeyCode::VK_STICKY_EXTRA4 - - -I use the virtual modifier key `EXTRA4`. This key is only intelligible inside -of KR4MB, but that's not a problem since I do all of the rest of the Greek -mapping inside there too. - -I wrote a little [Python script][cadetpy] to generate the rest of the mapping -for me. Here's the Greek layer mapping in its entirety: - - -What are all those `Option` modifiers doing in there? Well there's one more -step to making this work. In order to use this mapping you need to switch your -keyboard layout in OS X to the "Unicode Hex Input" layout under System -Preferences → Language and Text → Input Sources: - -![Switching to Unicode Hex Input](/media/images{{ parent_url }}/kb-hex.png) - -As far as I can tell this is exactly like QWERTY except that it also allows you -to type in Unicode characters directly by holding `Option` and typing in their -Unicode code point's hex value. That's how the mapping inserts the Unicode -characters for the Greek letters. - -If you want to do this with another keyboard layout I'm not sure how you could -do it. If you find a way let me know and I'll post it here. - -However, there's a problem. The Unicode Hex Input keyboard layout disables OS -X's `Option-Left` and `Option-Right` keyboard shortcuts for some reason I don't -even want to try to guess. Luckily they can be restored with another mapping: - - :::xml - - Restore [Shift-]Opt-(Left/Right) - The Unicode Hex Input keyboard layout disables these keys for some reason. - - space_cadet.fix_opt_arrows - - - --KeyToKey-- - KeyCode::CURSOR_LEFT, VK_OPTION | VK_SHIFT, - - KeyCode::B, - ModifierFlag::OPTION_L | ModifierFlag::SHIFT_L | ModifierFlag::CONTROL_L - - - --KeyToKey-- - KeyCode::CURSOR_LEFT, VK_OPTION, - - KeyCode::B, - ModifierFlag::OPTION_L | ModifierFlag::CONTROL_L - - - --KeyToKey-- - KeyCode::CURSOR_RIGHT, VK_OPTION | VK_SHIFT, - - KeyCode::F, - ModifierFlag::OPTION_L | ModifierFlag::SHIFT_L | ModifierFlag::CONTROL_L - - - --KeyToKey-- - KeyCode::CURSOR_RIGHT, VK_OPTION, - - KeyCode::F, - ModifierFlag::OPTION_L | ModifierFlag::CONTROL_L - - - -`Option-Delete` is also broken by Unicode Hex Input. I haven't figured out how -to fix this, but I don't really care about it because I have the following in my -`~/Library/KeyBindings/DefaultKeyBinding.dict` file which lets me use -`Control-w` just like at the command line: - - :::text - { - "^w"="deleteWordBackward:"; - } - -Welcome to the fiddly rabbit hole of keybinding customization! - -The Greek layer in my mapping pretty much matches the original Space Cadet -mapping as far as I can tell. Here's a table of the keys and their Greek -counterparts: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
KeyAloneShiftedNotes
aαΑAlpha
bβΒBeta
cχΧChi
dδΔDelta
eεΕEpsilon
fφΦPhi (p was taken and the ph sounds like an f)
gγΓGamma
hηΗEta
iιΙIota
jϑΘTheta (with one of the lowercase variants)
kκΚKappa
lλΛLambda (Lisp users rejoice!)
mμΜMu
nνΝNu
oοΟOmicron
pπΠPi
qθΘTheta (with the other lowercase variant)
rρΡRho
sσΣSigma (with one of the lowercase variants)
tτΤTau
uυΥUpsilon
vςΣSigma (with the other lowercase variant)
wωΩOmega (o was taken and the lowercase kind of looks like a w)
xξΞXi
yψΨPsi (p was taken and it looks a bit like a y)
zζΖZeta
- -[cadetpy]: https://github.com/sjl/dotfiles/blob/master/keyremap4macbook/cadet.py - -### Math - -The Space Cadet also had a `Top` key for typing the symbols on the top of the -keys, which were (I think) used in APL. I don't use APL, but when typing out -mathematical text it would be nice to have some symbols easily available. - -I've added a Math layer that functions similarly to the Greek layer. I chose -the right `Control` key to activate it. I won't go into detail about how it -works because it's the same as the Greek layer. Here's the code for it if you -want to use it yourself: - -I've currently only filled in the Math layer with keys I find useful, so there -are a lot of empty keys at the moment. I plan to add new ones as I discover -I want them. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
KeyAloneShiftedNotes
aAnd, Aleph
b
cComposition, Complex numbers
d
eElement of, Not element of
fIntegral (i was taken and it looks like an f)
g
h
iIntersection, Infinity
j
k
l
m
nNatural numbers
oOr
p
q
rRoot, Real numbers
s
t
uUnion
v
w
xXor
y
zIntegers
-¬Not (looks like a minus)
=±Not equal, Plus or minus (Shift-= is normally +)
/÷Division
,Less than or equal to (Shift-, is normally <)
.Greater than or equal to (Shift-. is normally >)
1
2
3
4
5
6
7
8×Times or cross product (Shift-8 is normally *)
9
0Null set
`Approximately equal to (Shift-` is normally ~)
[Proper subset of, Not proper subset of
]Proper superset of, Not proper superset of
Left
Right
Up/Down
- -In addition, the last five rows also support the `Control` key for more -variants: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
KeyCtrlCtrl-ShiftedNotes
[Subset of, Not subset of
]Superset of, Not superset of
Left
Right
Up/Down
- -Conclusion ----------- - -If you want my full KeyRemap4MacBook configuration you can always get [the -latest version from my dotfiles][dotkr]. - -I've done some crazy, fiddly things to my machine. Undoubtedly lots of people -on Hacker News will scoff and brag about how they only use the defaults. -I don't really care. Given that I spend 60 hours or more a week at the keyboard -if these things only save me 0.01% of my time they'll pay for themselves in -a year or two. Not to mention all the finger pain they'll prevent. Plus, -tinkering around and seeing how far you can push things is fun (sometimes)! - -I'm pretty satisfied with my hardware and layout. My dream would be to get -a custom keycap set with my key labels and the Space Cadet colors/typography -that would fit on a white Realforce. It would be absolutely beautiful, but I'm -sure it'd be far too expensive to make a single custom set. Oh well, maybe some -day! - -[dotkr]: https://github.com/sjl/dotfiles/blob/master/keyremap4macbook/private.xml - -{% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/10/a-modern-space-cadet.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2012/10/a-modern-space-cadet.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,1307 @@ ++++ +title = "A Modern Space Cadet" +snip = "Emulating a legendary keyboard." +date = 2012-10-03T09:55:00Z +draft = false + ++++ + +I spend a lot of my time (easily over 8 hours a day) at a keyboard. As you +might have guessed from my previous entries about [Vim][] and [Mutt][] I'm not +averse to spending a few hours to improve an environment I'm going to be +spending tens of thousands of hours in over the next few years, so it shouldn't +be a shock that my keyboard is something I've heavily tweaked. + +This post is about what I've done to make my typing experience more pleasant and +efficient. + +If you scoff at customization you won't enjoy this post. What if I have to use +someone else's machine? I can count on one hand the number of times I've had to +use someone else's machine ever since I got an iPhone five years ago. I *like* +customizing my machine to save me time. Yes, there may be some excessive stuff +here, but not only does it make me type faster, it's *fun*! + +{{% toc %}} + +[Vim]: /blog/2010/09/coming-home-to-vim/ +[Mutt]: /blog/2012/10/the-homely-mutt/ + +The Original +------------ + +There have been many, many keyboards produced in the world since the first ones +emerged. One of the most famous, at least in programming circles, is the [Space +Cadet Keyboard][space-cadet]. + +Originally used on Lisp machines and some other systems, it's not a keyboard for +the minimalist. There are four "bucky keys" (modifier keys): control, meta, +super, and hyper (plus shift, of course). There are also special keys that let +you type Greek letters and mathematical symbols. + +The Space Cadet is an example of a keyboard for someone not afraid to invest +some time to work faster. + +It's also absolutely gorgeous. The color scheme and typography is beautiful. + +This is the keyboard I used (loosely) as inspiration when crafting my current +setup. + +[space-cadet]: http://world.std.com/~jdostale/kbd/SpaceCadet.html + +Modern Hardware +--------------- + +I've tried a number of modern keyboards in the past few years. They're all +high-quality and more expensive than the $20 plastic toys that come with +desktops. But I spend 60 or more hours a week at a keyboard and maybe one hour +a week in my car, so I'm getting pretty good use out of the dollars I've put +into keyboards if you compare them to the cost of my car! + +I'll go through the keyboards I've used in chronological order. I'm not going +to write too much about the basics of mechanical keyboards and switches. If you +want to learn about that, [this post][ch] and [this guide][mech] are good places +to start. + +[ch]: http://www.codinghorror.com/blog/2010/10/the-keyboard-cult.html +[mech]: http://www.overclock.net/t/491752/mechanical-keyboard-guide + +### Apple Wireless Keyboard + +For a long time I used [Apple wireless keyboards][apple-wireless]. + +![Apple Wireless Keyboard](/media/images/blog/2012/10/kb-apple.jpg) + +They're light and compact, but still feel extremely solid thanks to their metal +construction. + +They also stay really clean because there gaps between keys are tiny, instead of +the funnel-shaped gaps of other keyboard that send dirt straight to the bottom. + +They're readily available at any Apple store, and of course they're wireless +which is great. + +They also feel exactly like the keyboards on Apple's laptops, so your muscle +memory is perfectly suited to either one if you switch between them often. + +Unfortunately typing on them is nowhere near as nice as the rest of the +keyboards in this list. Apple keyboards have (I think) only 2mm of travel, but +you have to bottom-out the keys to register the keypresses. + +[apple-wireless]: http://www.apple.com/keyboard/ + +### Das Silent Ultimate + +The first mechanical keyboard I got was the [Das Silent Ultimate][das-silent]. + +![Das Silent Keyboard](/media/images/blog/2012/10/kb-das.jpg) + +(This photo is actually my Das Clicky since I don't have the Silent any more, +but they're exactly the same externally.) + +Compared to the Apple keyboards the switches on the Das Silent (Cherry Brown +switches) feel far softer. You're not smashing your fingers against metal on +every keypress. + +Overall the Das Silent isn't too bad. It's built like a tank so I have no doubt +it'd last forever. It also has the option of blank keys, which I prefer. + +I didn't stick with this keyboard for long. First, its glossy finish looks good +when it's in the box and terrible forever after that. It's way too easy to get +dirty. + +Something also just feels "off" about the key switches to me, especially after +I've tried the other boards in this list. They feel, for lack of a better term, +"mushy" to me. + +[das-silent]: http://www.daskeyboard.com/model-s-ultimate-silent/ + +### Das Clicky Ultimate + +The next keyboard I tried was the [Das Clicky Ultimate][das-clicky]. + +![Das Clicky Keyboard](/media/images/blog/2012/10/kb-das.jpg) + +This is exactly the same as the Das Silent except for the switches, which are +the "clicky" Cherry Blues. These feel *way* better to me than the browns. The +mushiness I described for the Silent is completely gone. + +They are not kidding when they say it is loud. If you're typing a lot expect to +have a constant background noise of clicks. But to me the sound is soothing, +especially once you get into a rhythm of typing for extended periods. Your +coworkers might disagree, so tread carefully. + +The Das Clicky still has that godawful glossy finish, unfortunately. After +a while I decided to try out what I had heard people raving about: Topre +switches. + +[das-clicky]: http://www.daskeyboard.com/model-s-ultimate/ + +### Happy Hacking Keyboard Professional 2 + +The [Happy Hacking Keyboard][hhkb] was my next keyboard. It's much more +expensive than the Dases were, so it was a tough call, but I'm glad I got it. + +![Happy Hacking Keyboard](/media/images/blog/2012/10/kb-hhkb.jpg) + +The HHKB uses Topre key switches, which to me feel like Cherry Browns done +right. There's no "click" like the Blues, but instead of the mush of the +Browns you get a satisfying "thunk" or "sshhhunk" on each press. It's a great +balance: you have a nice satisfying sound that's less annoying for other people. + +The finish on this keyboard is perfect. It is the most unassuming, elegant, +wonderful matte finish I've seen on any keyboard. The plastic is just textured +enough to feel solid under your fingers (not slippery like Macbook keys) but not +enough to feel like you're typing on sandpaper. + +If you're short on space, or want to carry your keyboard with you, the HHKB is +for you. It's *tiny*, but still feels solid and not flimsy at all. + +I have two main complaints about the HHKB. First, there's two spaces on the +bottom row where they could have put a modifier key but didn't. The space on +the right has the logo, which is fair enough, but the one on the left is simply +blank for no good reason. + +My other complaint is the idiotic default placement of the `fn` key and the +arrow keys. I don't mind having to use `fn` to get to the arrows as +a concession to a compact design, but why on earth would you place them so that +you have to chord on a *single hand* to hit them? + +This problem would be completely solved if the `fn` key were moved from its +current position (right of the right shift key) down to the empty lower-left +corner. Then you could use your left hand to hold `fn` while your right worked +the arrow keys, no single-hand contortion required. + +So after a while with the HHKB I got sick of the arrow key problem. I loved the +Topre switches though, so the next choice was pretty obvious. + +[hhkb]: http://elitekeyboards.com/products.php?sub=pfu_keyboards,hhkbpro2&pid=pdkb400bn + +### Topre Realforce 103 + +The Topre Realforce 103UB was my next keyboard, and is the one I'm +still using to this day. + +![Topre Realforce Keyboard](/media/images/blog/2012/10/kb-realforce.jpg) + +I believe the model I bought is now discontinued, but it's been replaced by the +[104UB][topre] which is exactly the same except for an extra key on the right +side. If I could swap my current keyboard for one of those I would. In fact, +if anyone wants to buy my used 103UB for a bit of a discount I'd be totally +willing to sell it to get the 104UB. + +Anyway, this keyboard has a few disadvantages. First, unlike every other +keyboard here (except the wireless Apple ones) *it is not a USB hub*. This +isn't a huge deal (I have external USB hubs), but it did come as a big surprise. + +Second, it's big. Really big. If you don't have *plenty* of room on your desk +you might want to look at the [tenkeyless][topre-tenkeyless] varieties which +ditch the number pad section to save space. I have room and I love having +a number pad, so I got the behemoth-sized model. + +Also for some reason I can't fathom there's no way to get blank, black keycaps +for this keyboard. You can get blank Topre keycaps in lime green, pink, yellow, +and lots of other colors, but not a simple black! + +Now for the good parts. This thing is built like a tank. If you needed to +defend your home from an intruder, it would make an acceptable blunt weapon with +which to do so (and you'd probably be able to plug it right back in and start +typing again). + +The finish is the same as on the HHKB (gorgeous), but I have to take a point off +for the non-blank keycaps. I don't need to look at the keys while I type, so +why marr the luxurious finish with lettering? + +The Realforce also uses Topre switches, but it has another trick up its sleeve +that makes typing on it even nicer than the HHKB. The version I have is the +"variable-weighted" one, which means that the keys under your weaker fingers +take less force to press than the ones under your stronger fingers. This is +fantastic for long sessions of typing. Instead of my pinkies getting tired my +fingers seem to all tire out at the same rate now. Topre does make a "uniform" +version that doesn't have this trick, but I really like variable weight keys +myself. + +[topre]: http://elitekeyboards.com/products.php?sub=topre_keyboards,rf104&pid=xf11t0 +[topre-tenkeyless]: http://elitekeyboards.com/products.php?sub=topre_keyboards,rftenkeyless + +### Verdict + +Right now I'm using the Topre Realforce for my day to day work. I've fallen in +love with how the Topre switches feel (I prefer them over all the others except +*maybe* the Cherry Blues for the sound) and the Happy Hacking Keyboard is just +too cramped. If I were short on space the HHKB would be great, but I have +plenty of room on my desk, so why not make use of it? + +Here's a photo of all the keyboards (along with a 13" Macbook Air) so you can +see the differences in size. + +![Keyboard Size Comparison](/media/images/blog/2012/10/kb-size.jpg) + +Modern Software +--------------- + +Now that I've talked about the hardware it's time for the software. I use OS +X exclusively, so everything I say here is OS X specific. I'm sure there are +Windows and Linux equivalents somewhere though. + +I use three software programs that, together, give me just about unlimited +flexibility in customizing how my keyboard works in OS X. + +### Keyboard Maestro + +[Keyboard Maestro][] is a utility for binding macros to keyboard shortcuts in OS +X. It may not have the best user interface, but it can do a *lot*, and once you +set up your shortcuts you never have to look at it again. It's $36. + +Right now I use it for application switching, with a few twists I'll cover later +on. + +I switched to [Keymando][] for a while. It was great being able to configure it +in a programming language with a plain text file I could easily work with in +version control, but compared to Keyboard Maestro it's far slower and far +buggier. I'm now back to only using Keyboard Maestro. + +[Keyboard Maestro]: http://www.keyboardmaestro.com/main/ +[Keymando]: http://keymando.com/ + +### PCKeyboardHack + +[PCKeyboardHack][] lets you map one key on your keyboard to another at a very +low level. For example, you can change just the right Option key to send `F19` +instead. I use this for one single key which you'll see in the next section. +It's free and [open source][pc-git]. + +[PCKeyboardHack]: http://pqrs.org/macosx/keyremap4macbook/pckeyboardhack.html.en +[pc-git]: https://github.com/tekezo/PCKeyboardHack + +### KeyRemap4MacBook + +[KeyRemap4MacBook][] is how I do the bulk of my keyboard customizations. Like +all of the other applications its user interface is horrible, but it can do damn +near *anything* you might want. You could probably replace Keyboard Maestro +with it, at least for the kind of stuff I do. It's also free and [open +source][kr-git]. + +The bulk of what I describe in this post is going to use KeyRemap4MacBook. I'm +not going to give you a tutorial in it here — read its documentation if you +want to learn how to use the things I'm going to show. + +[KeyRemap4MacBook]: http://pqrs.org/macosx/keyremap4macbook/index.html.en +[kr-git]: https://github.com/tekezo/KeyRemap4MacBook + +Control/Escape +-------------- + +These first few mappings aren't directly from the Space Cadet keyboard, but they +were inspired by its spirit of making an efficient tool for text editing. + +The Capslock key on modern keyboard has become the punchline of many a joke, and +for good reason: there's no reason to dedicate such an important key to +a function like capslock. Many people rebind it to a more useful key like +Control, Option, Escape, or Backspace. I rebound it to Control for a while and +then realized that with KeyRemap4MacBook I could get even more mileage out of +it. + +The important thing I thought of one day is that it's possible to divide keys on +the keyboard into three groups: + +* Keys you hold down to change how *other* keys behave, but that (usually) don't + do anything if you use them on their own (like Shift and Control). +* Keys that you press and release but don't want to "repeat" as you hold them + (like Escape or Insert). +* Keys that you sometimes press and release, but sometimes want to repeat (like + holding Space to insert a bunch of spaces, or Backspace to kill a bunch of + characters). + +Can you see where this is heading? The last group is pretty normal, but the +first two groups are special. Specifically: there are two different ways to use +them and they're each only useful in one of those ways. + +This means that we can *combine* them onto a single key without losing any +useful functionality! + +I'm clearly not the first one to think of this, because KR4MB includes built-in +support for creating these kinds of mappings. + +First I've mapped Capslock to Control at the OS X level by going into the +Keyboard preference pane in System Preferences, clicking the Modifier Keys +button in the lower right, selecting the keyboard in the dropdown list (this is +surprisingly easy to miss), and changing the Capslock setting: + +![Changing Capslock to Control in OS X](/media/images/blog/2012/10/kb-caps.png) + +Then I've selected the following premade option in KeyRemap4MacBook: + +> Control\_L to Control\_L +> (+ when you type Control\_L only, send Escape) + +Now the Capslock key on the keyboard does the following: + +1. If held down and pressed with another key, it acts like Control. +2. If pressed and released on its own, it acts like Escape. + +That's two extremely important keys (at least for Vim users) on a single key in +one of the prime locations on the keyboard! No more awkward stretches! + +If you try this and want to force yourself to learn to use it, disable the +normal escape key. You'll learn fast. + +**Update:** I forgot to mention that if you use this (and the Shift +improvements below) you'll probably want to change the "\[Key Overlaid +Modifier\] Timeout" setting in the KR4MB preferences: + +![Key Overlaid Timeout](/media/images/blog/2012/10/kb-key-overlaid.png) + +This setting controls the maximum length of a keypress that will register as +"single press". It's easier to understand with an example. The default is +1000ms, which means that if you press and hold the Capslock key for 900ms and +release it, it will count as Escape. If you hold it for 1001ms before +releasing, it counts as pressing and releasing Control instead. + +If you're finding that you're sometimes holding Control out of habit and then +releasing it, but it's still firing as an Escape, you may want to turn this +setting down to a lower value. I have mine set to 300ms. + +This is a lot more noticeable with the Shift mappings I'll describe shortly, so +keep it in the back of your mind if you try those out. + +Hyper +----- + +Modern OS X supports four "modifier" keys: Control, Option, Command, and Shift. +The Space Cadet keyboard had five: Control, Meta, Super, Hyper, and Shift. + +Shift and Control map to each other, and OS X's Option key is pretty much the +same as Meta (in fact I think Emacs users usually use Alt as Meta). +I arbitrarily decided that Command was the OS X equivalent of Super. That left +Hyper. + +Since OS X doesn't natively support a fifth modifier key I had to come up with +a way to fake it. + +First, I realized that because I use Capslock for Control I had no use for the +*real* left Control key. So the first step was to remap that to something else +distinct from the Capslock-version of Control. + +This required PCKeyboardHack. I don't think there's a way to do it in +KeyRemap4MacBook, because by the time KR4MB sees the keypress it can't tell if +it came from the Capslock key or the real Control key (due to how they were +changed in the previous section). PCKeyboardHack, however, *can*, so I remapped +left Control to keycode `80`: + +![Remapping Left Control in PCKeyboardHack](/media/images/blog/2012/10/kb-pck.png) + +Keycode `80` is the `F19` key. My keyboard doesn't have an `F19` key so it +doesn't conflict with anything. Now I can simply remap `F19` in KR4MB just like +any other key. + +To create a "pseudo-Hyper" modifier, I remapped this key to be the equivalent of +holding down *all four other modifiers* by adding the following to my +`private.xml` KR4MB configuration file: + +```xml + + Remap Left Control to Hyper + OS X doesn't have a Hyper. This maps Left Control to Control + Shift + Option + Command. + + space_cadet.left_control_to_hyper + + + --KeyToKey-- + KeyCode::F19, + + KeyCode::COMMAND_L, + ModifierFlag::OPTION_L | ModifierFlag::SHIFT_L | ModifierFlag::CONTROL_L + + +``` + +As far as I can tell, no keyboard shortcuts in OS X or any apps use all four +modifier keys (how would you normally press them all, anyway?). But many +programs, like Keyboard Maestro, let you define your own shortcuts. So now I've +got an entire key as a "namespace" all to myself for my own shortcuts! + +This is really nice. I don't have to worry about "shadowing" existing shortcuts +anywhere and it's only a single modifier to press. + +Currently I use this "namespace" for application switching. Instead of using +the normal `Command-Tab` switcher, I have shortcuts for each individual app +I use frequently. For example, `Hyper-k` switches to Firefox, `Hyper-y` +switches to Twitter, and so on. This is better than `Command-Tab`ing because +I don't have to worry about how many times I need to press it. `Hyper-k` +*always* goes to Firefox no matter what, so I can easily burn that into my +muscle memory. + +There's one more little trick I use in Keyboard Maestro. It's specific to how +I work, but I'm sure some of you will still find it handy. + +I pretty much always keep two iTerm 2 windows open. The first contains +a [tmux][] session with one window split into two panes. One pane holds +[weechat][weechat-prog] for IRC, the other holds [Mutt][mutt-prog] for email. +I keep this on my laptop screen at all times while I do other things on my +external monitor. + +The second iTerm window is almost always fullscreened, and contains a tmux +session with whatever I'm working on. The number of windows and panes varies +wildly. + +What I wanted was a way to bind `Hyper-i` and `Hyper-m` to directly focus my IRC +and mail panes, and `Hyper-j` to directly focus the second, "general-purpose" +iTerm window. + +The solution came in two parts. First I configured tmux so that `prefix N` +would select the nth pane in the current window by adding the following to +`~/.tmux.conf`: + +```text +bind 1 select-pane -t 1 +bind 2 select-pane -t 2 +bind 3 select-pane -t 3 +bind 4 select-pane -t 4 +bind 5 select-pane -t 5 +bind 6 select-pane -t 6 +bind 7 select-pane -t 7 +bind 8 select-pane -t 8 +bind 9 select-pane -t 9 +``` + +My tmux prefix is `Control-f`, so now pressing `Control-f 1` will go to pane 1, +and so on. Then I configured Keyboard Maestro like so: + +![Keyboard Maestro IRC Config](/media/images/blog/2012/10/kb-irc.png) + +This binds `Control-Shift-Option-Command-i` (which is just `Hyper-i`) to do the +following: + +1. Focus iTerm 2. +2. Send a `Command-Option-1` keystroke, which will focus the first iTerm + 2 window (I simply make sure I always open my mail/irc window first). +3. Send the `Control-f` keystroke. +3. Send the `1` keystroke, which together with the previous one tells tmux to + switch to pane 1. + +The mail shortcut is similar, of course, and the general-purpose terminal one is +even simpler. + +So now I've got a free modifier key that won't conflict with anything, and I've +got some very easy-to-type shortcuts I can burn into my fingers for switching +applications quickly. Awesome. + +[tmux]: http://tmux.sourceforge.net/ +[weechat-prog]: http://www.weechat.org/ +[mutt-prog]: http://www.mutt.org/ + +Better Shifting +--------------- + +The Shift keys are another of those keys that are only useful with other keys, +so it's only natural that they were also on the list of keys to optimize. +First, however, I took a detour to correct a bad habit of mine. + +### Shift Key Training Wheels + +I've been typing for most of my life, but I never really learned to do it +correctly. I can touch type, of course, but sometimes I use the wrong fingers +for certain keys. + +My most egregious offense was that I always used the left Shift key. Even when +typing `X` I'd hold the left Shift key with my left pinky and hit the `x` with +my left index finger, which pulled my hand off the home row and generally felt +awkward. + +One day I decided I was tired of doing this the wrong way and resolved to fix +myself. The easiest way to break a bad habit is to make it harder or impossible +to do, so I created a custom KeyRemap4MacBook setting that *disables* the +keypress when you use the incorrect Shift key with a letter. Here it is in +full: + +```xml + + Use the correct shift keys. + private.correct_shift_keys + + --KeyToKey-- KeyCode::Q, ModifierFlag::SHIFT_L, KeyCode::VK_NONE + --KeyToKey-- KeyCode::W, ModifierFlag::SHIFT_L, KeyCode::VK_NONE + --KeyToKey-- KeyCode::E, ModifierFlag::SHIFT_L, KeyCode::VK_NONE + --KeyToKey-- KeyCode::R, ModifierFlag::SHIFT_L, KeyCode::VK_NONE + --KeyToKey-- KeyCode::T, ModifierFlag::SHIFT_L, KeyCode::VK_NONE + --KeyToKey-- KeyCode::A, ModifierFlag::SHIFT_L, KeyCode::VK_NONE + --KeyToKey-- KeyCode::S, ModifierFlag::SHIFT_L, KeyCode::VK_NONE + --KeyToKey-- KeyCode::D, ModifierFlag::SHIFT_L, KeyCode::VK_NONE + --KeyToKey-- KeyCode::F, ModifierFlag::SHIFT_L, KeyCode::VK_NONE + --KeyToKey-- KeyCode::G, ModifierFlag::SHIFT_L, KeyCode::VK_NONE + --KeyToKey-- KeyCode::Z, ModifierFlag::SHIFT_L, KeyCode::VK_NONE + --KeyToKey-- KeyCode::X, ModifierFlag::SHIFT_L, KeyCode::VK_NONE + --KeyToKey-- KeyCode::C, ModifierFlag::SHIFT_L, KeyCode::VK_NONE + --KeyToKey-- KeyCode::V, ModifierFlag::SHIFT_L, KeyCode::VK_NONE + + --KeyToKey-- KeyCode::Y, ModifierFlag::SHIFT_R, KeyCode::VK_NONE + --KeyToKey-- KeyCode::U, ModifierFlag::SHIFT_R, KeyCode::VK_NONE + --KeyToKey-- KeyCode::I, ModifierFlag::SHIFT_R, KeyCode::VK_NONE + --KeyToKey-- KeyCode::O, ModifierFlag::SHIFT_R, KeyCode::VK_NONE + --KeyToKey-- KeyCode::P, ModifierFlag::SHIFT_R, KeyCode::VK_NONE + --KeyToKey-- KeyCode::H, ModifierFlag::SHIFT_R, KeyCode::VK_NONE + --KeyToKey-- KeyCode::J, ModifierFlag::SHIFT_R, KeyCode::VK_NONE + --KeyToKey-- KeyCode::K, ModifierFlag::SHIFT_R, KeyCode::VK_NONE + --KeyToKey-- KeyCode::L, ModifierFlag::SHIFT_R, KeyCode::VK_NONE + --KeyToKey-- KeyCode::SEMICOLON, ModifierFlag::SHIFT_R, KeyCode::VK_NONE + --KeyToKey-- KeyCode::N, ModifierFlag::SHIFT_R, KeyCode::VK_NONE + --KeyToKey-- KeyCode::M, ModifierFlag::SHIFT_R, KeyCode::VK_NONE + --KeyToKey-- KeyCode::COMMA, ModifierFlag::SHIFT_R, KeyCode::VK_NONE + --KeyToKey-- KeyCode::DOT, ModifierFlag::SHIFT_R, KeyCode::VK_NONE + --KeyToKey-- KeyCode::QUOTE, ModifierFlag::SHIFT_R, KeyCode::VK_NONE + +``` + +It took me about four hours of normal computer use to unlearn over two decades +of wrong typing. It's amazing how fast muscle memory can adjust when you simply +*force* it to. + +Once I fixed myself I disabled this setting, because it also makes you to use +the "correct" Shift key when doing keyboard shortcuts, and sometimes those are +easer to do with the "wrong" Shift. + +If you only use one Shift key I'd really encourage you to try this. It spreads +out the wear over both pinkies and feels much better. Your hands will thank +you. + +### Shift Parentheses + +Now that I was using the right Shift keys, it was time to revisit mapping +something else onto them. One pair of keys stood out as a perfect candidate: the +left and right parentheses. + +Parentheses are common in most of the programming languages in use today (some +more than others). If you counted I bet you'd find them more common than square +or curly brackets in the majority of your code, and yet they're shoved away in +the horrible homes of `Shift-9` and `Shift-0`. + +I decided to try out remapping my Shift keys to work like so: + +* When held while pressing other keys, act like Shift. +* When pressed and released on their own, type an opening or closing parenthesis + (left and right shift respectively). + +This means I can type parentheses with a single, unchorded keypress. After +using it for a while I absolutely love it. Here's the KeyRemap4MacBook setting +for it: + +```xml + + Shifts to Parentheses + Shifts, when pressed alone, type parentheses. When used with other keys they're normal shifts. + + private.shifts_to_parens + + + --KeyOverlaidModifier-- KeyCode::SHIFT_R, ModifierFlag::SHIFT_R | ModifierFlag::NONE, KeyCode::SHIFT_R, KeyCode::KEY_0, ModifierFlag::SHIFT_L + --KeyOverlaidModifier-- KeyCode::SHIFT_L, ModifierFlag::SHIFT_L | ModifierFlag::NONE, KeyCode::SHIFT_L, KeyCode::KEY_9, ModifierFlag::SHIFT_R + + + --KeyToKey-- KeyCode::SHIFT_L, ModifierFlag::SHIFT_R, KeyCode::KEY_0, ModifierFlag::SHIFT_L, KeyCode::KEY_9, ModifierFlag::SHIFT_L + --KeyToKey-- KeyCode::SHIFT_R, ModifierFlag::SHIFT_L, KeyCode::KEY_9, ModifierFlag::SHIFT_L, KeyCode::KEY_0, ModifierFlag::SHIFT_L + + + --KeyToKey-- KeyCode::SPACE, ModifierFlag::SHIFT_R, KeyCode::KEY_0, ModifierFlag::SHIFT_L, KeyCode::SPACE + +``` + +As you can see there's actually a bit more to the mapping, because I was finding +some minor usability glitches in the bare setting. + +Occasionally I'll accidentally type a parenthesis when meaning to do something +else, but it happens so infrequently and the benefits are so great that I'd +absolutely recommend this to anyone who programs a lot. + +Key Layers +---------- + +Now we're getting to things that are directly inspired by the Space Cadet +keyboard. If you don't type a lot of mathematical characters you may not want +to bother with these, but since I've been taking the [Introduction to +Mathematical Thinking][coursera] class on Coursera these bindings have made it +very pleasant to type out my homework. + +[coursera]: https://www.coursera.org/course/maththink + +### Greek + +The Space Cadet had a separate key for typing Greek letters, and I've decided to +do the same. I don't know *exactly* how that modifier key worked, but here's +how my version works: + +* Press and release the "Greek" key to go into "Greek mode" for one character. +* Type the appropriate Latin letter to get the Greek character for that key + (capital letters and lowercase are distinct). + +For example, typing `Greek`, then `w` results in ω (omega), while `Greek`, then +`W` results in Ω (capital omega). + +If you want to actually type out full Greek words, this isn't ideal because you +have to press the `Greek` key between every letter. But for my purposes +(variables in math) it's perfect. + +There are two parts to implementing this in KeyRemap4MacBook. First I needed to +pick a `Greek` key. I never use the Command, Option, and Control keys to the +right of Space (another bad habit I should probably fix one day), so I chose the +right Option key for this. + +If you *do* use the modifiers on the right you might prefer to pick a different +key for this, like `Insert` or `Print Screen`. + +Also, since I'm using the Realforce which is a PC keyboard, that key is actually +called the `PC Application` key. I'm sure you can figure out how to adjust the +mapping if necessary. + +Here's the first part of the mapping, which binds the `Greek` key: + +```xml + + Greek Layer + Right PC Application key activates the Greek key layer. + + space_cadet.greek_layer + + + --KeyToKey-- + KeyCode::PC_APPLICATION, + KeyCode::VK_STICKY_EXTRA4 + +``` + +I use the virtual modifier key `EXTRA4`. This key is only intelligible inside +of KR4MB, but that's not a problem since I do all of the rest of the Greek +mapping inside there too. + +I wrote a little [Python script][cadetpy] to generate the rest of the mapping +for me. Here's the Greek layer mapping in its entirety: + + +What are all those `Option` modifiers doing in there? Well there's one more +step to making this work. In order to use this mapping you need to switch your +keyboard layout in OS X to the "Unicode Hex Input" layout under System +Preferences → Language and Text → Input Sources: + +![Switching to Unicode Hex Input](/media/images/blog/2012/10/kb-hex.png) + +As far as I can tell this is exactly like QWERTY except that it also allows you +to type in Unicode characters directly by holding `Option` and typing in their +Unicode code point's hex value. That's how the mapping inserts the Unicode +characters for the Greek letters. + +If you want to do this with another keyboard layout I'm not sure how you could +do it. If you find a way let me know and I'll post it here. + +However, there's a problem. The Unicode Hex Input keyboard layout disables OS +X's `Option-Left` and `Option-Right` keyboard shortcuts for some reason I don't +even want to try to guess. Luckily they can be restored with another mapping: + +```xml + + Restore [Shift-]Opt-(Left/Right) + The Unicode Hex Input keyboard layout disables these keys for some reason. + + space_cadet.fix_opt_arrows + + + --KeyToKey-- + KeyCode::CURSOR_LEFT, VK_OPTION | VK_SHIFT, + + KeyCode::B, + ModifierFlag::OPTION_L | ModifierFlag::SHIFT_L | ModifierFlag::CONTROL_L + + + --KeyToKey-- + KeyCode::CURSOR_LEFT, VK_OPTION, + + KeyCode::B, + ModifierFlag::OPTION_L | ModifierFlag::CONTROL_L + + + --KeyToKey-- + KeyCode::CURSOR_RIGHT, VK_OPTION | VK_SHIFT, + + KeyCode::F, + ModifierFlag::OPTION_L | ModifierFlag::SHIFT_L | ModifierFlag::CONTROL_L + + + --KeyToKey-- + KeyCode::CURSOR_RIGHT, VK_OPTION, + + KeyCode::F, + ModifierFlag::OPTION_L | ModifierFlag::CONTROL_L + + +``` + +`Option-Delete` is also broken by Unicode Hex Input. I haven't figured out how +to fix this, but I don't really care about it because I have the following in my +`~/Library/KeyBindings/DefaultKeyBinding.dict` file which lets me use +`Control-w` just like at the command line: + +```text +{ + "^w"="deleteWordBackward:"; +} +``` + +Welcome to the fiddly rabbit hole of keybinding customization! + +The Greek layer in my mapping pretty much matches the original Space Cadet +mapping as far as I can tell. Here's a table of the keys and their Greek +counterparts: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
KeyAloneShiftedNotes
aαΑAlpha
bβΒBeta
cχΧChi
dδΔDelta
eεΕEpsilon
fφΦPhi (p was taken and the ph sounds like an f)
gγΓGamma
hηΗEta
iιΙIota
jϑΘTheta (with one of the lowercase variants)
kκΚKappa
lλΛLambda (Lisp users rejoice!)
mμΜMu
nνΝNu
oοΟOmicron
pπΠPi
qθΘTheta (with the other lowercase variant)
rρΡRho
sσΣSigma (with one of the lowercase variants)
tτΤTau
uυΥUpsilon
vςΣSigma (with the other lowercase variant)
wωΩOmega (o was taken and the lowercase kind of looks like a w)
xξΞXi
yψΨPsi (p was taken and it looks a bit like a y)
zζΖZeta
+ +[cadetpy]: https://github.com/sjl/dotfiles/blob/master/keyremap4macbook/cadet.py + +### Math + +The Space Cadet also had a `Top` key for typing the symbols on the top of the +keys, which were (I think) used in APL. I don't use APL, but when typing out +mathematical text it would be nice to have some symbols easily available. + +I've added a Math layer that functions similarly to the Greek layer. I chose +the right `Control` key to activate it. I won't go into detail about how it +works because it's the same as the Greek layer. Here's the code for it if you +want to use it yourself: + +I've currently only filled in the Math layer with keys I find useful, so there +are a lot of empty keys at the moment. I plan to add new ones as I discover +I want them. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
KeyAloneShiftedNotes
aAnd, Aleph
b
cComposition, Complex numbers
d
eElement of, Not element of
fIntegral (i was taken and it looks like an f)
g
h
iIntersection, Infinity
j
k
l
m
nNatural numbers
oOr
p
q
rRoot, Real numbers
s
t
uUnion
v
w
xXor
y
zIntegers
-¬Not (looks like a minus)
=±Not equal, Plus or minus (Shift-= is normally +)
/÷Division
,Less than or equal to (Shift-, is normally <)
.Greater than or equal to (Shift-. is normally >)
1
2
3
4
5
6
7
8×Times or cross product (Shift-8 is normally *)
9
0Null set
`Approximately equal to (Shift-` is normally ~)
[Proper subset of, Not proper subset of
]Proper superset of, Not proper superset of
Left
Right
Up/Down
+ +In addition, the last five rows also support the `Control` key for more +variants: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
KeyCtrlCtrl-ShiftedNotes
[Subset of, Not subset of
]Superset of, Not superset of
Left
Right
Up/Down
+ +Conclusion +---------- + +If you want my full KeyRemap4MacBook configuration you can always get [the +latest version from my dotfiles][dotkr]. + +I've done some crazy, fiddly things to my machine. Undoubtedly lots of people +on Hacker News will scoff and brag about how they only use the defaults. +I don't really care. Given that I spend 60 hours or more a week at the keyboard +if these things only save me 0.01% of my time they'll pay for themselves in +a year or two. Not to mention all the finger pain they'll prevent. Plus, +tinkering around and seeing how far you can push things is fun (sometimes)! + +I'm pretty satisfied with my hardware and layout. My dream would be to get +a custom keycap set with my key labels and the Space Cadet colors/typography +that would fit on a white Realforce. It would be absolutely beautiful, but I'm +sure it'd be far too expensive to make a single custom set. Oh well, maybe some +day! + +[dotkr]: https://github.com/sjl/dotfiles/blob/master/keyremap4macbook/private.xml + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/10/caves-of-clojure-07-1.html --- a/content/blog/2012/10/caves-of-clojure-07-1.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,328 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "The Caves of Clojure: Part 7.1" - snip: "Region mapping." - created: 2012-10-15 09:50:00 - %} - -{% block article %} - -This post is part of an ongoing series. If you haven't already done so, you -should probably start at [the beginning][]. - -This entry corresponds to the beginning of [post seven in Trystan's -tutorial][trystan-tut]. - -If you want to follow along, the code for the series is [on Bitbucket][bb] and -[on GitHub][gh]. Update to the `entry-07-1` tag to see the code as it stands -after this post. - -It's been a while since the last post, but I've been taking care of things and -hopefully should be able to write a bit more now. - -This post is going to be short. It'll cover a relatively self-contained but -interesting bit of Trystan's seventh post. The rest of it will be covered in -the next entry. - -[the beginning]: /blog/2012/07/caves-of-clojure-01/ -[trystan-tut]: http://trystans.blogspot.com/2011/09/roguelike-tutorial-07-z-levels-and.html -[bb]: http://bitbucket.org/sjl/caves/ -[gh]: http://github.com/sjl/caves/ - -[TOC] - -Summary -------- - -In Trystan's seventh post he adds vertical levels and stairs connecting them. -I'm going to cover the first part of that now: mapping out regions. - -There's been a bit of refactoring since my last post which I'm not going to -cover. If you want to see what changed, diff the tags in your VCS of choice. - -Region Mapping --------------- - -In order to decide where to place stairs, Trystan maps out "regions" of -contiguous, walkable tiles after he generates and smooths the world. I'm going -to do the same thing. - -My goal is to create a "region map", which is a map of coordinates to region -numbers. For example, consider the following tiny world map: - - :::text - ..##.. - ..#... - ..#.## - ..#.#. - -There are three distinct regions in this map: - - :::text - 11##22 - 11#222 - 11#2## - 11#2#3 - -So the region map would look like: - - :::clojure - ; x y region - {[0 0] 1 - [1 0] 1 - [4 0] 2 - [5 0] 2 - ... - [5 3] 3} - -This makes it easy to tell which region a particular tile is in (if any). - -As usual, I'll start with a few helper functions. These two functions are just -for convenience and readability: - - :::clojure - (def all-coords - (let [[cols rows] world-size] - (for [x (range cols) - y (range rows)] - [x y]))) - - (defn get-tile-from-level [level [x y]] - (get-in level [y x] (:bound tiles))) - -The `all-coords` function simply returns a lazy sequence of `[x y]` coordinates -representing every coordinate in a level. - -`get-tile-from-level` encapsulates the act of pulling out a tile from a level -given an `[x y]` coordinate. This is helpful because of the way I'm storing -tiles (note the ugly `[y x]`). - -Next up is a function that filters a set of coordinates to only contain those -that are actually walkable in the given level (i.e.: those that don't contain -a wall tile): - - :::clojure - (defn filter-walkable - "Filter the given coordinates to include only walkable ones." - [level coords] - (set (filter #(tile-walkable? (get-tile-from-level level %)) - coords))) - -This uses the `get-tile-from-level` function as well as `tile-walkable?` from -`world.core`. - -Next is a function to take a coordinate and return which of its neighboring -coordinates are walkable: - - :::clojure - (defn walkable-neighbors - "Return the neighboring coordinates walkable from the given coord." - [level coord] - (filter-walkable level (neighbors coord))) - -This one is almost trivial, but I like building up functions in small steps like -this because it's easier for me to read. - -Now we come to a function with a bit more meat. This is the core of the "flood -fill" algorithm I'm going to use to fill in the region map. - - :::clojure - (defn walkable-from - "Return all coordinates walkable from the given coord (including itself)." - [level coord] - (loop [walked #{} - to-walk #{coord}] - (if (empty? to-walk) - walked - (let [current (first to-walk) - walked (conj walked current) - to-walk (disj to-walk current) - candidates (walkable-neighbors level current) - to-walk (union to-walk (difference candidates walked))] - (recur walked to-walk))))) - -In a nutshell, this function loops over two sets: `walked` and `to-walk`. - -Each iteration it grabs a coordinate from `to-walk` and sticks it into the -`walked` set. It then finds all the coordinates it can walk to from that -coordinate using a helper function. It uses `clojure.set/difference` to -determine which of those are new (i.e.: still need to be walked) and sticks them -into the `to-walk` set. Then it recurs. - -The code for this is surprisingly simple and easy to read. It's mostly just -shuffling things between sets. Eventually the `to-walk` set will be empty and -`walked` will contain all the coordinates that we want. - -Finally comes the function to create the region map for an entire level: - - :::clojure - (defn get-region-map - [level] - (loop [remaining (filter-walkable level all-coords) - region-map {} - n 0] - (if (empty? remaining) - region-map - (let [next-coord (first remaining) - next-region-coords (walkable-from level next-coord)] - (recur (difference remaining next-region-coords) - (into region-map (map vector - next-region-coords - (repeat n))) - (inc n)))))) - -This function also uses Clojure sets to its advantage. Once again, I loop -over a couple of variables. - -`remaining` is a set containing all the coordinates whose regions has not yet -been determined. - -Each iteration it pulls off one of the remaining coordinates. Note that I'm -using `first` to do this. `remaining` is a set, which is unordered, so `first` -effectively could return any element in the set. For this loop that doesn't -matter, but it's important to be aware of if you're going to use the same -strategy. - -After pulling off a coordinate, it finds all coordinates walkable from that -coordinate with the `walkable-from` flood-fill function. It removes all of -those from the `remaining` set, shoves them into the region map, and increments -the region number before recurring. - -Visualization -------------- - -I'm going to save the rest of Trystan's seventh post for another entry, but -since this one ended up pretty short I'm also going to go over visualizing the -region map I've just created. - -First I need to generate the region map when I create the world, and attach it -to the world itself so we can access it from other places: - - :::clojure - (defn random-world [] - (let [world (->World (random-tiles) {}) - world (nth (iterate smooth-world world) 3) - world (populate-world world) - world (assoc world :regions (get-region-map (:tiles world)))] - world)) - -The last line in the `let` is where it gets generated. It's pretty -straightforward. - -I'd like to be able to toggle the visualization of regions off and on, so I'm -going to introduce a new concept to the game: "debug flags". - -I updated the `Game` record to include a slot for these flags: - - :::clojure - (defrecord Game [world uis input debug-flags]) - -I then updated the `new-game` function to initialize them (currently there's -only one) to default values: - - :::clojure - (defn new-game [] - (map->Game {:world nil - :uis [(->UI :start)] - :input nil - :debug-flags {:show-regions false}})) - -The user needs a way to toggle them. For now I'll just bind it to a key. In -the future I could make a debug UI with a nice menu. - - :::clojure - (defmethod process-input :play [game input] - (case input - :enter (assoc game :uis [(->UI :win)]) - :backspace (assoc game :uis [(->UI :lose)]) - \q (assoc game :uis []) - - \h (update-in game [:world] move-player :w) - \j (update-in game [:world] move-player :s) - - ; ... - - \R (update-in game [:debug-flags :show-regions] not) - - game)) - -Now when the user presses `R` (Shift and R) it will toggle the state of the -`:show-regions` debug flag in the game. - -All that's left is to actually *draw* the regions somehow. First, we only want -to do this if `:show-regions` is `true`. I edited the `:play` UI's drawing -function to do this: - - :::clojure - (defmethod draw-ui :play [ui game screen] - (let [world (:world game) - {:keys [tiles entities regions]} world - player (:player entities) - [cols rows] (s/get-size screen) - vcols cols - vrows (dec rows) - origin (get-viewport-coords game (:location player) vcols vrows)] - (draw-world screen vrows vcols origin tiles) - ; ****************** - (when (get-in game [:debug-flags :show-regions]) - (draw-regions screen regions vrows vcols origin)) - ; ****************** - (doseq [entity (vals entities)] - (draw-entity screen origin vrows vcols entity)) - (draw-hud screen game) - (draw-messages screen (:messages player)) - (highlight-player screen origin player))) - -The marked lines are the only new ones. I'm going to draw regions after/above -the world tiles (so they'll show up at all) but before/below the entities (so we -can still see what's going on). - -Drawing the regions is fairly simple, if a bit tedious: - - :::clojure - (defn draw-regions [screen region-map vrows vcols [ox oy]] - (letfn [(get-region-glyph [region-number] - (str - (nth - "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" - region-number)))] - (doseq [x (range ox (+ ox vcols)) - y (range oy (+ oy vrows))] - (let [region-number (region-map [x y])] - (when region-number - (s/put-string screen (- x ox) (- y oy) - (get-region-glyph region-number) - {:fg :blue})))))) - -For now, bad things will happen if we have more than 62 regions in a single -level. In practice I usually end up with about 20 to 30, so it's not a big -deal. - -To sum up this function: it iterates through every coordinate in the level -that's displayed in the viewport, looks up its region number in the region map, -and draws the appropriate letter if it has a region number. - -Results -------- - -Now that I've got a way to visualize regions it becomes much easier to check -whether they're getting set correctly. Here's an example of what it looks like -when you toggle `:show-regions` with `R`: - -![Screenshot without Regions](/media/images{{ parent_url }}/caves-07-1-1.png) - -![Screenshot with Regions](/media/images{{ parent_url }}/caves-07-1-2.png) - -As you can see, the small, closed off areas have their own numbers, while the -larger regions sprawl across the map. - -You can view the code [on GitHub][result-code] if you want to see the end -result. - -[result-code]: https://github.com/sjl/caves/tree/entry-07-1/src/caves - -The next article will finish Trystan's seventh post by adding multiple z-levels -to the caves. - -{% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/10/caves-of-clojure-07-1.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2012/10/caves-of-clojure-07-1.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,339 @@ ++++ +title = "The Caves of Clojure: Part 7.1" +snip = "Region mapping." +date = 2012-10-15T09:50:00Z +draft = false + ++++ + +This post is part of an ongoing series. If you haven't already done so, you +should probably start at [the beginning][]. + +This entry corresponds to the beginning of [post seven in Trystan's +tutorial][trystan-tut]. + +If you want to follow along, the code for the series is [on Bitbucket][bb] and +[on GitHub][gh]. Update to the `entry-07-1` tag to see the code as it stands +after this post. + +It's been a while since the last post, but I've been taking care of things and +hopefully should be able to write a bit more now. + +This post is going to be short. It'll cover a relatively self-contained but +interesting bit of Trystan's seventh post. The rest of it will be covered in +the next entry. + +[the beginning]: /blog/2012/07/caves-of-clojure-01/ +[trystan-tut]: http://trystans.blogspot.com/2011/09/roguelike-tutorial-07-z-levels-and.html +[bb]: http://bitbucket.org/sjl/caves/ +[gh]: http://github.com/sjl/caves/ + +{{% toc %}} + +Summary +------- + +In Trystan's seventh post he adds vertical levels and stairs connecting them. +I'm going to cover the first part of that now: mapping out regions. + +There's been a bit of refactoring since my last post which I'm not going to +cover. If you want to see what changed, diff the tags in your VCS of choice. + +Region Mapping +-------------- + +In order to decide where to place stairs, Trystan maps out "regions" of +contiguous, walkable tiles after he generates and smooths the world. I'm going +to do the same thing. + +My goal is to create a "region map", which is a map of coordinates to region +numbers. For example, consider the following tiny world map: + +```text +..##.. +..#... +..#.## +..#.#. +``` + +There are three distinct regions in this map: + +```text +11##22 +11#222 +11#2## +11#2#3 +``` + +So the region map would look like: + +```clojure +; x y region +{[0 0] 1 + [1 0] 1 + [4 0] 2 + [5 0] 2 + ... + [5 3] 3} +``` + +This makes it easy to tell which region a particular tile is in (if any). + +As usual, I'll start with a few helper functions. These two functions are just +for convenience and readability: + +```clojure +(def all-coords + (let [[cols rows] world-size] + (for [x (range cols) + y (range rows)] + [x y]))) + +(defn get-tile-from-level [level [x y]] + (get-in level [y x] (:bound tiles))) +``` + +The `all-coords` function simply returns a lazy sequence of `[x y]` coordinates +representing every coordinate in a level. + +`get-tile-from-level` encapsulates the act of pulling out a tile from a level +given an `[x y]` coordinate. This is helpful because of the way I'm storing +tiles (note the ugly `[y x]`). + +Next up is a function that filters a set of coordinates to only contain those +that are actually walkable in the given level (i.e.: those that don't contain +a wall tile): + +```clojure +(defn filter-walkable + "Filter the given coordinates to include only walkable ones." + [level coords] + (set (filter #(tile-walkable? (get-tile-from-level level %)) + coords))) +``` + +This uses the `get-tile-from-level` function as well as `tile-walkable?` from +`world.core`. + +Next is a function to take a coordinate and return which of its neighboring +coordinates are walkable: + +```clojure +(defn walkable-neighbors + "Return the neighboring coordinates walkable from the given coord." + [level coord] + (filter-walkable level (neighbors coord))) +``` + +This one is almost trivial, but I like building up functions in small steps like +this because it's easier for me to read. + +Now we come to a function with a bit more meat. This is the core of the "flood +fill" algorithm I'm going to use to fill in the region map. + +```clojure +(defn walkable-from + "Return all coordinates walkable from the given coord (including itself)." + [level coord] + (loop [walked #{} + to-walk #{coord}] + (if (empty? to-walk) + walked + (let [current (first to-walk) + walked (conj walked current) + to-walk (disj to-walk current) + candidates (walkable-neighbors level current) + to-walk (union to-walk (difference candidates walked))] + (recur walked to-walk))))) +``` + +In a nutshell, this function loops over two sets: `walked` and `to-walk`. + +Each iteration it grabs a coordinate from `to-walk` and sticks it into the +`walked` set. It then finds all the coordinates it can walk to from that +coordinate using a helper function. It uses `clojure.set/difference` to +determine which of those are new (i.e.: still need to be walked) and sticks them +into the `to-walk` set. Then it recurs. + +The code for this is surprisingly simple and easy to read. It's mostly just +shuffling things between sets. Eventually the `to-walk` set will be empty and +`walked` will contain all the coordinates that we want. + +Finally comes the function to create the region map for an entire level: + +```clojure +(defn get-region-map + [level] + (loop [remaining (filter-walkable level all-coords) + region-map {} + n 0] + (if (empty? remaining) + region-map + (let [next-coord (first remaining) + next-region-coords (walkable-from level next-coord)] + (recur (difference remaining next-region-coords) + (into region-map (map vector + next-region-coords + (repeat n))) + (inc n)))))) +``` + +This function also uses Clojure sets to its advantage. Once again, I loop +over a couple of variables. + +`remaining` is a set containing all the coordinates whose regions has not yet +been determined. + +Each iteration it pulls off one of the remaining coordinates. Note that I'm +using `first` to do this. `remaining` is a set, which is unordered, so `first` +effectively could return any element in the set. For this loop that doesn't +matter, but it's important to be aware of if you're going to use the same +strategy. + +After pulling off a coordinate, it finds all coordinates walkable from that +coordinate with the `walkable-from` flood-fill function. It removes all of +those from the `remaining` set, shoves them into the region map, and increments +the region number before recurring. + +Visualization +------------- + +I'm going to save the rest of Trystan's seventh post for another entry, but +since this one ended up pretty short I'm also going to go over visualizing the +region map I've just created. + +First I need to generate the region map when I create the world, and attach it +to the world itself so we can access it from other places: + +```clojure +(defn random-world [] + (let [world (->World (random-tiles) {}) + world (nth (iterate smooth-world world) 3) + world (populate-world world) + world (assoc world :regions (get-region-map (:tiles world)))] + world)) +``` + +The last line in the `let` is where it gets generated. It's pretty +straightforward. + +I'd like to be able to toggle the visualization of regions off and on, so I'm +going to introduce a new concept to the game: "debug flags". + +I updated the `Game` record to include a slot for these flags: + +```clojure +(defrecord Game [world uis input debug-flags]) +``` + +I then updated the `new-game` function to initialize them (currently there's +only one) to default values: + +```clojure +(defn new-game [] + (map->Game {:world nil + :uis [(->UI :start)] + :input nil + :debug-flags {:show-regions false}})) +``` + +The user needs a way to toggle them. For now I'll just bind it to a key. In +the future I could make a debug UI with a nice menu. + +```clojure +(defmethod process-input :play [game input] + (case input + :enter (assoc game :uis [(->UI :win)]) + :backspace (assoc game :uis [(->UI :lose)]) + \q (assoc game :uis []) + + \h (update-in game [:world] move-player :w) + \j (update-in game [:world] move-player :s) + + ; ... + + \R (update-in game [:debug-flags :show-regions] not) + + game)) +``` + +Now when the user presses `R` (Shift and R) it will toggle the state of the +`:show-regions` debug flag in the game. + +All that's left is to actually *draw* the regions somehow. First, we only want +to do this if `:show-regions` is `true`. I edited the `:play` UI's drawing +function to do this: + +```clojure +(defmethod draw-ui :play [ui game screen] + (let [world (:world game) + {:keys [tiles entities regions]} world + player (:player entities) + [cols rows] (s/get-size screen) + vcols cols + vrows (dec rows) + origin (get-viewport-coords game (:location player) vcols vrows)] + (draw-world screen vrows vcols origin tiles) + ; ****************** + (when (get-in game [:debug-flags :show-regions]) + (draw-regions screen regions vrows vcols origin)) + ; ****************** + (doseq [entity (vals entities)] + (draw-entity screen origin vrows vcols entity)) + (draw-hud screen game) + (draw-messages screen (:messages player)) + (highlight-player screen origin player))) +``` + +The marked lines are the only new ones. I'm going to draw regions after/above +the world tiles (so they'll show up at all) but before/below the entities (so we +can still see what's going on). + +Drawing the regions is fairly simple, if a bit tedious: + +```clojure +(defn draw-regions [screen region-map vrows vcols [ox oy]] + (letfn [(get-region-glyph [region-number] + (str + (nth + "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + region-number)))] + (doseq [x (range ox (+ ox vcols)) + y (range oy (+ oy vrows))] + (let [region-number (region-map [x y])] + (when region-number + (s/put-string screen (- x ox) (- y oy) + (get-region-glyph region-number) + {:fg :blue})))))) +``` + +For now, bad things will happen if we have more than 62 regions in a single +level. In practice I usually end up with about 20 to 30, so it's not a big +deal. + +To sum up this function: it iterates through every coordinate in the level +that's displayed in the viewport, looks up its region number in the region map, +and draws the appropriate letter if it has a region number. + +Results +------- + +Now that I've got a way to visualize regions it becomes much easier to check +whether they're getting set correctly. Here's an example of what it looks like +when you toggle `:show-regions` with `R`: + +![Screenshot without Regions](/media/images/blog/2012/10/caves-07-1-1.png) + +![Screenshot with Regions](/media/images/blog/2012/10/caves-07-1-2.png) + +As you can see, the small, closed off areas have their own numbers, while the +larger regions sprawl across the map. + +You can view the code [on GitHub][result-code] if you want to see the end +result. + +[result-code]: https://github.com/sjl/caves/tree/entry-07-1/src/caves + +The next article will finish Trystan's seventh post by adding multiple z-levels +to the caves. + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/10/the-homely-mutt.html --- a/content/blog/2012/10/the-homely-mutt.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1319 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "The Homely Mutt" - snip: "Sparrow's dead? Why not try Mutt?" - created: 2012-10-01 10:30:00 - %} - -{% block article %} - -Now that [Sparrow][] is [effectively dead][sparrow-dead] many of its users will -be looking for a new email client. If you're not afraid of the terminal you may -want to give [Mutt][] a try. - -Mutt certainly isn't the prettiest email client around, and its -setup/configuration process is one of the ugliest out there. But once you get -it set up it's got a lot of advantages over many other email clients. - -In this post I'll show you how to set up Mutt on OS X like I do. - -[Sparrow]: http://sparrowmailapp.com/ -[sparrow-dead]: http://www.theverge.com/2012/7/20/3172222/google-buys-sparrow-mail -[Mutt]: http://www.mutt.org/ - -[TOC] - -How I Use Email ---------------- - -This setup is going to be specific to the way I work with email. Notably: - -* I have a Google Apps account that provides my steve@stevelosh.com email - address. -* I have many other email addresses, but they all simply forward to my main one. -* All mail I send comes from steve@stevelosh.com. -* I store my contacts in the OS X address book. -* All email comes into my inbox (or to a folder for a specific mailing list). -* Once I'm done with an email, I remove it from my inbox and it lives in the - "All Mail" archive. I don't sort email into folders after it arrives. -* I sometimes read email offline and mark it for deletion, then sync that - deletion back to the server once I get online again. -* Sometimes I write email without an internet connection and send it once I get - connected again. - -My email setup is tailored around those requirements, so that's what it does -best. Mutt is very configurable though, so if you work differently you can -probably bend it to make it do what you want. - -In particular, extending this setup to work with multiple email accounts -wouldn't be too much trouble. I used to work with two separate accounts until -I said "screw it, I'll just use the one". - -Other Guides and Resources --------------------------- - -I've used a lot of other guides to figure out how to get this giant Rube -Goldberg machine of an email client working. Here are a few of them: - -* -* -* -* -* -* -* -* -* -* - -Overview --------- - -I'm going to give it to you straight: getting this whole contraption set up is -going to take at least an hour from start to finish, not counting the time it'll -take to download all of your email and install stuff. Set aside an evening if -you're serious about this. - -It's an investment, and you might not want to make it. If not, go use -Thunderbird, er, Sparrow, er, I don't know, the Gmail web interface or -something. - -Mutt on its own doesn't do very much, so we're going to combine it with a few -other things to get the job done. Here's a bird's eye view of what it'll look -like when we're done: - -![Diagram](/media/images{{ parent_url }}/what-the-mutt.png) - -If this diagram doesn't make you run screaming you might just be masochistic -enough to make it through the initial setup of Mutt. If you do, you'll be -rewarded with email bliss that won't go away when Google or Facebook decide to -toss some money around. - -Getting Email -------------- - -First thing's first: we're going to pull down our email from Gmail to our local -machine. All of it. It'll take a while the first time you sync, but has a few -benefits. - -### Why Local Email? - -Having a local copy of all of your email means you've always got access to it, -no matter where you are. Looking for that one person's address they emailed you -six years ago when you're trying to find their house and you don't have an -internet connection? No problem, it's on your hard drive. - -This also acts as a backup in case Google ever decides to kill your Gmail -account. It'll be stored in a common format that a lot of programs can read, so -you've got a safety net. And the email is stored as normal files, so if you use -something like Time Machine or [Backblaze][] that's yet another free backup. - -In this setup **all of your email is stored as plain text**. If you want it -encrypted just use OS X's full-disk encryption and you're set. - -I use [offlineimap][] to pull email down from Gmail and get it on my hard drive. -Offlineimap will also sync any changes you make to this local copy back up to -Gmail. - -[offlineimap]: http://offlineimap.org/ -[Backblaze]: http://www.backblaze.com/partner/af3574 - -### The Alternative - -You may not care as much about reading your email offline as I do. If you can -tolerate always needing an internet connection to read your mail, you can skip -this painful section and follow [this guide][roma] instead. - -You'll probably still find the other sections of this post interesting though. - -[roma]: http://empt1e.blogspot.com/2009/10/using-mutt-with-gmail-imap-complete.html - -### Installing offlineimap - -I've gone through a number of laptops in the past few years, and each time -I spend a painful half hour or so screwing around with the latest version of -offlineimap's backwards-incompatible changes. - -If you're determined to run the latest version of offlineimap, you can install -it with pip or something. If you just want to download your fucking email and -get on with your life, you can follow the instructions I've laid out for you -here: - - :::text - git clone git://github.com/spaetz/offlineimap.git - cd offlineimap - git checkout 679c491c56c981961e18aa43b31955900491d7a3 - python setup.py install - -That's the version I'm using. It works. You can use a newer one if you want, -but expect to spend some time figuring out how to fix the configuration in this -post to work with whatever breaking changes have been made since then. The last -time I tried this I got to rewrite all my nametrans stuff. That was fun. - -### Configuring offlineimap - -Once you've got offlineimap installed, you'll need to create -a `~/.offlineimaprc` file. You can keep it in your dotfiles repo and symlink it -into place if you want. Here's a sample to get you started: - - :::text - [general] - ui = TTY.TTYUI - accounts = SteveLosh - pythonfile=~/.mutt/offlineimap.py - fsync = False - - [Account SteveLosh] - localrepository = SteveLosh-Local - remoterepository = SteveLosh-Remote - status_backend = sqlite - postsynchook = notmuch new - - [Repository SteveLosh-Local] - type = Maildir - localfolders = ~/.mail/steve-stevelosh.com - nametrans = lambda folder: {'drafts': '[Gmail]/Drafts', - 'sent': '[Gmail]/Sent Mail', - 'flagged': '[Gmail]/Starred', - 'trash': '[Gmail]/Trash', - 'archive': '[Gmail]/All Mail', - }.get(folder, folder) - - [Repository SteveLosh-Remote] - maxconnections = 1 - type = Gmail - remoteuser = steve@stevelosh.com - remotepasseval = get_keychain_pass(account="steve@stevelosh.com", server="imap.gmail.com") - realdelete = no - nametrans = lambda folder: {'[Gmail]/Drafts': 'drafts', - '[Gmail]/Sent Mail': 'sent', - '[Gmail]/Starred': 'flagged', - '[Gmail]/Trash': 'trash', - '[Gmail]/All Mail': 'archive', - }.get(folder, folder) - folderfilter = lambda folder: folder not in ['[Gmail]/Trash', - 'Nagios', - 'Django', - 'Flask', - '[Gmail]/Important', - '[Gmail]/Spam', - ] - -It's kind of long, so let's go through it line by line and see what's going on. - - :::text - [general] - ui = TTY.TTYUI - accounts = SteveLosh - pythonfile=~/.mutt/offlineimap.py - fsync = False - -First we tell offlineimap to use the `TTY.TTYUI` ui. Yes, this program that -syncs your email has multiple user interfaces. I guess if you can't decide what -color the bikeshed should be you can just build a whole bunch of bikesheds -instead. - -Then we specify the accounts. There's only one because as I said before: -I only use a single email account that all my addresses forward to. If you -wanted to have many, you'd change this line. - -The `pythonfile` is just a file that offlineimap will parse (as Python) before -loading the rest of the config, so you can define custom helper functions more -easily. We'll see more of this later. - -We're also telling offlineimap that it doesn't need to fsync after every single -operation. This will speed things up, and since it's just a local copy it's -typically not a big deal if we lose an email here and there from a crash (it'll -just be synced the next time anyway). - - :::text - [Account SteveLosh] - localrepository = SteveLosh-Local - remoterepository = SteveLosh-Remote - status_backend = sqlite - -This next section hooks up a few things. First, it tells offlineimap which -local and remote repositories to use for the account. Manual configuration -instead of sane defaults is a recurring theme we'll see throughout this process. - -Hey, I titled the entry "The *Homely* Mutt" for a reason. - -We're also going to use a SQLite-based cache for this account. If you don't -already have SQLite you'll want to get it with `brew install sqlite`. - - :::text - [Repository SteveLosh-Local] - type = Maildir - localfolders = ~/.mail/steve-stevelosh.com - nametrans = lambda folder: {'drafts': '[Gmail]/Drafts', - 'sent': '[Gmail]/Sent Mail', - 'flagged': '[Gmail]/Starred', - 'trash': '[Gmail]/Trash', - 'archive': '[Gmail]/All Mail', - }.get(folder, folder) - -Now we're getting to the meat of the configuration. This "local repository" is -going to be the mail as it sits on our hard drive. We're going to use the -[Maildir format][maildir] because it plays nicely with Mutt (and tons of other -stuff). - -Then we specify the path where we're going to keep the mail. This is going to -take a lot of space if you've got a lot of mail. Attachments are downloaded -too. When I said you're getting an offline copy of all your email I meant *all* -of it. - -I think offlineimap needs the `~/.mail` directory created for it. It's been -a while since I did this, so I might be wrong, but if it complains about not -being able to access the mail folders just go ahead and `mkdir ~/.mail`. - -Next we have the craziest part of the configuration: name translation. - -Here's the issue: offlineimap needs to know how to translate the names of -folders on the IMAP server to folder names on your hard drive. - -Also, Gmail doesn't actually use *folders* but its own concept called "labels". -But since the IMAP protocol doesn't know about labels, it fakes them by making -them appear to be folders. - -User-created labels in Gmail (like "Mercurial" or "Clients") will appear as -folders with those names through IMAP. - -Built-in, special Gmail folders have names that start with `[Gmail]/`. We need -to turn those into something sane for our hard drive, so that's what this -nametrans setting is for. It's a Python function that takes the remote folder -name and returns the name that should be used on your local hard drive. - -Yes, you read that right. This is Python code embedded in the right hand side -of an INI file's setting assignment. I am not fucking with you, this is -seriously how you do it. Go ahead and crack open that beer now. - -So the "Sent Mail" folder in your Gmail account will be synced to -`~/.mail/steve-stevelosh.com/sent`. Cool. - -(No, I don't know what would happen if you created a label called `[Gmail]/All -Mail` in Gmail. If you try please let me know, but I take no responsibility if -it ends with all your email being deleted.) - -[maildir]: https://en.wikipedia.org/wiki/Maildir - - :::text - [Repository SteveLosh-Remote] - maxconnections = 1 - type = Gmail - remoteuser = steve@stevelosh.com - remotepasseval = get_keychain_pass(account="steve@stevelosh.com", server="imap.gmail.com") - realdelete = no - nametrans = lambda folder: {'[Gmail]/Drafts': 'drafts', - '[Gmail]/Sent Mail': 'sent', - '[Gmail]/Starred': 'flagged', - '[Gmail]/Trash': 'trash', - '[Gmail]/All Mail': 'archive', - }.get(folder, folder) - folderfilter = lambda folder: folder not in ['[Gmail]/Trash', - 'Nagios', - 'Django', - 'Flask', - '[Gmail]/Important', - '[Gmail]/Spam', - ] - -Finally, the home stretch. The last section described the folder on our local -hard drive, and this one describes our Gmail account. - -First, we tell offlineimap to only ever use a single connection at a time. You -can try increasing this number for better performance, but in my experience -Google is not afraid to enforce its rate limits and would cut me off fairly -often when I tried that. Just leave it at one if you want to be safe. - -Next is the type. Luckily offlineimap provides a `Gmail` type that handles -a lot of the craziness that is Gmail's IMAP setup. Nice. - -Then we have the username. Nothing special here, except that if you have -a non-apps account (i.e.: an actual vanilla Gmail account) you may or may not -need to include the `@gmail.com` in the username. I don't know. If one doesn't -work, just try the other. - -Next we have `remotepasseval`. This is a bit of Python code (drink!) that -should return the password for the account. - -What is this `get_keychain_pass` function? Well, remember when we saw the -`pythonfile` setting back in the general section? It's a function defined in -there. I'll talk about that in the next section, for now just accept that it -works. - -Next we set `realdelete` to no. If this is set to yes, then deleting an email -in your inbox would actually delete it entirely. When you set it to no, then -deleting an email from your inbox (or any label's folder) will leave it in -Gmail's All Mail. - -If you want to really delete an email, you'll need to delete it from All Mail -(which is named archive on our local filesystem, remember?). I feel like this -is a good compromise. I rarely care about actually deleting mail, given that -I have many unused gigabytes available on Gmail. - -Next we have another nametrans setting. This is a Python function (drink!) just -like the one for the local repository, except it goes in the other direction. -It takes the name of a local folder and returns the name of the folder on the -IMAP server. Knowing this, it should be easy to understand this setting. - -Finally, we have `folderfilter`. This is a Python function (drink!) that takes -a **remote** folder name and returns `True` if that folder should be synced, or -`False` if it should **not** be synced. I've chosen to skip syncing my Spam and -Trash folders, as well as a few mailing list labels I don't check all that -often. Customize this to your own taste. - -### Retrieving Passwords - -We're almost ready, but there's one more thing we need to do, and that's -implement a secure way for offlineimap to get access to our Gmail password. - -If you don't care too much about security, you *can* configure offlineimap with -a plaintext password right in the config file. But don't do that. It'll only -take a minute to do this securely. - -First, you need to add your Gmail password into your OS X keychain. Open the -Keychain Access app and press the `+` button: - -![Keychain 1](/media/images{{ parent_url }}/keychain-1.png) - -Then fill out the form. The "Keychain Item Name" should be -`http://imap.gmail.com`. The "Account Name" should be your email address. The -password should be your password: - -![Keychain 2](/media/images{{ parent_url }}/keychain-2.png) - -Press "Add". Now repeat the process for the SMTP server. The "Keychain Item -Name" should be `smtp://smtp.gmail.com`. The "Account Name" should be your -email address. The password should be your password: - -![Keychain 3](/media/images{{ parent_url }}/keychain-3.png) - -Now we need to create the `offlineimap.py` file we pointed offlineimap to -earlier. It needs to contain the `get_keychain_pass` function, which takes an -`account` and `server` and return the password. Here's the file I'm using: - - :::python - #!/usr/bin/python - import re, subprocess - def get_keychain_pass(account=None, server=None): - params = { - 'security': '/usr/bin/security', - 'command': 'find-internet-password', - 'account': account, - 'server': server, - 'keychain': '/Users/sjl/Library/Keychains/login.keychain', - } - command = "sudo -u sjl %(security)s -v %(command)s -g -a %(account)s -s %(server)s %(keychain)s" % params - output = subprocess.check_output(command, shell=True, stderr=subprocess.STDOUT) - outtext = [l for l in output.splitlines() - if l.startswith('password: ')][0] - - return re.match(r'password: "(.*)"', outtext).group(1) - -In a nutshell, it uses `/usr/bin/security` to retrieve the password. Read -through the code if you're curious. - -This is not *completely* secure, but it's better than having your password in -a plaintext file in your home directory. - -Whew! Time to actually run this thing and pull down our email! - -### Running offlineimap - -Assuming everything is in place, open a terminal and run offlineimap: - - offlineimap - -Go read a book, because this is going to pull down all the email (with -attachments) in any folders you didn't exclude in the config file. - -**If there's an error, stop and figure out what went wrong**. Remember, -offlineimap is a *two-way* sync, so there's always the possibility it'll eat -your email if you seriously mess something up! I wish it had a -`--dont-touch-remote` option you could use as a safety net for the original -sync, but it doesn't, so be careful! - -In the future you can use `offlineimap -q` to run it in "quick mode". It'll -perform fewer checks but will generally be much faster. - -If you want to set up offlineimap to run every 5 minutes or so, you can use -`launchd`. `cron` does not work for some reason. I'm not entirely sure why. - -Personally I actually *like* having to press a key to fetch new mail. It's less -of a distraction than having new mail rolling in all the time. I can get new -email when I'm ready to actually look at it, rather than having it nagging me -all the time. - -The great part about offlineimap is that once you've got it configured and it's -successfully run once, it's pretty much rock solid from then on. You can run it -often, `Ctrl-c` it, put the laptop to sleep in the middle of a run, or `kill -9` -it, and it still won't lose emails. On the next sync it'll fix anything that's -missing. - -Mutt! ------ - -Now that you've got your email on your computer, it's finally time to start -using Mutt itself! - -### Installing - -Mutt can be installed in a bunch of different ways, but the easiest is through -Homebrew: - - :::text - brew install mutt --sidebar-patch - -The sidebar patch is a third-party patch that adds a sidebar to Mutt. I don't -know why it's not in core Mutt because it's insanely useful. Oh well, at least -Homebrew makes it simple to get. - -That's pretty much it for installation, but don't get too relaxed because you're -far from done. - -### Configuring - -Mutt is *very* configurable. This is great once you've become a power user and -want to mold it to your will, but terrible when you're just getting started. - -Mutt settings are kept in a `~/.muttrc` file. If this file doesn't exist Mutt -will look for `~/.mutt/muttrc` (note the lack of a dot in the filename), so you -can put it there if you prefer. - -Let's start by creating a basic `~/.muttrc` piece by piece (a lot of this was -taken from [this article][pris]). - -Once you've got a bit of Mutt under your belt you'll want to read [the -documentation][muttdoc] for these settings, but for now just use them to keep -things sane. - -[pris]: http://pbrisbin.com/posts/two_accounts_in_mutt -[muttdoc]: http://www.mutt.org/doc/manual/manual-6.html - - :::text - # Paths ---------------------------------------------- - set folder = ~/.mail # mailbox location - set alias_file = ~/.mutt/alias # where to store aliases - set header_cache = ~/.mutt/cache/headers # where to store headers - set message_cachedir = ~/.mutt/cache/bodies # where to store bodies - set certificate_file = ~/.mutt/certificates # where to store certs - set mailcap_path = ~/.mutt/mailcap # entries for filetypes - set tmpdir = ~/.mutt/temp # where to keep temp files - set signature = ~/.mutt/sig # my signature file - -Here we tell Mutt where to find the various folders it needs. - - :::text - # Basic Options -------------------------------------- - set wait_key = no # shut up, mutt - set mbox_type = Maildir # mailbox type - set timeout = 3 # idle time before scanning - set mail_check = 0 # minimum time between scans - unset move # gmail does that - set delete # don't ask, just do - unset confirmappend # don't ask, just do! - set quit # don't ask, just do!! - unset mark_old # read/new is good enough for me - set beep_new # bell on new mails - set pipe_decode # strip headers and eval mimes when piping - set thorough_search # strip headers and eval mimes before searching - -These are some basic options to make Mutt behave a bit more sanely. - - :::text - # Sidebar Patch -------------------------------------- - set sidebar_delim = ' │' - set sidebar_visible = yes - set sidebar_width = 24 - color sidebar_new color221 color233 - -These options are specific to the sidebar patch. - - :::text - # Status Bar ----------------------------------------- - set status_chars = " *%A" - set status_format = "───[ Folder: %f ]───[%r%m messages%?n? (%n new)?%?d? (%d to delete)?%?t? (%t tagged)? ]───%>─%?p?( %p postponed )?───" - -This gives us a pretty status bar with the information we care about (and none -of the stuff we don't). - - :::text - # Header Options ------------------------------------- - ignore * # ignore all headers - unignore from: to: cc: date: subject: # show only these - unhdr_order * # some distros order things by default - hdr_order from: to: cc: date: subject: # and in this order - -These options hide some of the extra email headers we don't care about when -viewing and composing email. - -Now it's time to fill on our account details: - - :::text - # Account Settings ----------------------------------- - - # Default inbox. - set spoolfile = "+steve-stevelosh.com/INBOX" - - # Alternate email addresses. - alternates sjl@pculture.org still\.?life@gmail.com steve@ladyluckblues.com steve@pculture.org - - # Mailboxes to show in the sidebar. - mailboxes +steve-stevelosh.com/INBOX \ - +steve-stevelosh.com/vim \ - +steve-stevelosh.com/clojure \ - +steve-stevelosh.com/python \ - +steve-stevelosh.com/mercurial \ - +steve-stevelosh.com/archive \ - +steve-stevelosh.com/sent \ - +steve-stevelosh.com/drafts \ - - # Other special folders. - set mbox = "+steve-stevelosh.com/archive" - set postponed = "+steve-stevelosh.com/drafts" - -Most of those should be self-explanatory. Fill in the appropriate values for -your mail folder(s). - -We'll add more as we go through the next few sections, but that's enough to get -us started. - -### Running - -Now that you've got Mutt configured you can run it: - - :::sh - mutt - -I like to always be in my `~/Desktop` folder when in Mutt, so that when I save -emails or attachments they go there by default. I have a little shell function -set up that `cd`s there for me before running Mutt: - - :::sh - alias mutt 'cd ~/Desktop && mutt' - -If you run the [new fish shell][fish], this is going to cause problems later -(long story, but it's related to the `read` builtin). Do yourself a favor and -head those confusing issues off at the pass with a fish function: - - :::text - function mutt - bash --login -c 'cd ~/Desktop; /usr/local/bin/mutt' $argv; - end - -Remember that if you use another shell like this you'll want to set up any -aliases and your `PATH` for that shell properly (probably identically to your -main shell). - -[fish]: http://ridiculousfish.com/shell/ - -Reading Email -------------- - -Once you start Mutt you should be looking at a list of the email in your inbox. -If so: congratulations! If not: stop and figure out what went wrong. - -### The Index - -When viewing a folder, Mutt presents you with a list of your email. This view -is called the "index": - -![Mutt's Index](/media/images{{ parent_url }}/mutt-index.png) - -This entry isn't meant be a guide to setting up Mutt on OS X. For a full guide -on how to *use* Mutt, you can Google around for some tutorials, or just learn as -you go with `?`. The `?` key will show you a list of all the keys you can use -wherever you currently are, and what they do. - -Let's add a few lines to our `~/.muttrc` to make the index view behave a bit -more nicely: - - :::text - # Index View Options --------------------------------- - set date_format = "%m/%d" - set index_format = "[%Z] %D %-20.20F %s" - set sort = threads # like gmail - set sort_aux = reverse-last-date-received # like gmail - set uncollapse_jump # don't collapse on an unread message - set sort_re # thread based on regex - set reply_regexp = "^(([Rr][Ee]?(\[[0-9]+\])?: *)?(\[[^]]+\] *)?)*" - -I won't go into what those do here. You can read the documentation if you're -curious. - -Quit and rerun Mutt to see your changes. Mutt is a very lightweight program so -this should be fast. - -Let's also add a few key bindings in the index to make it easier to use: - - :::text - # Index Key Bindings --------------------------------- - bind index gg first-entry - bind index G last-entry - - bind index R group-reply - bind index sync-mailbox - bind index collapse-thread - - # Ctrl-R to mark all as read - macro index \Cr "T~UN." "mark all messages as read" - - # Sync email - macro index O "offlineimap" "run offlineimap to sync all mail" - macro index o "offlineimap -qf INBOX" "run offlineimap to sync inbox" - - # Saner copy/move dialogs - macro index C "?" "copy a message to a mailbox" - macro index M "?" "move a message to a mailbox" - -Remember to quit and rerun Mutt for them to take effect. - -We're going to use `j` and `k` to move around, so we may as well support Vim -keys like `gg` and `G` too. We'll use `R` for reply all, since that comes in -handy fairly often. `Ctrl-R` will mark all messages in the current folder as -read. - -Don't worry if you don't understand how all these bindings and macros work right -now. You can read the documentation later. - -The `tab` key is going to "commit" changes we've made in Mutt (like deleting an -email) to our local Maildir folder. Once those changes are in the Maildir -folder offlineimap will sync them to the server the next time it runs. This is -nice because it lets us recover if we accidentally do something stupid like -deleting the wrong email. - -**Note:** Mutt will also sync changes for a folder when you switch to -a different folder, and when you quit Mutt, so be aware of those. - -The `space` key will toggle collapsing of threads, which can be convenient when -viewing mailing lists (or any conversations with many messages). - -The `o` and `O` keys will run offlineimap to sync mail. Like I said before, -I prefer having to press a button to grab mail instead of it constantly grabbing -and nagging me. `o` will sync only the inbox (fast), and `O` will sync -everything (much slower). - -Finally we rebind `C` and `M` to perform the same operations they usually do, -but in a more user-friendly manner. - -While we're at it, let's add a way to navigate around the sidebar so we can -switch folders: - - :::text - # Sidebar Navigation --------------------------------- - bind index,pager sidebar-next - bind index,pager sidebar-prev - bind index,pager sidebar-open - -We're binding the `up` and `down` arrow keys to switch between folders, and -`right` to "enter" a folder. Give it a try. - -We don't need the arrows because we can navigate with `j` and `k`, but if you -prefer to rebind them to something else feel free. - -Practice moving between folders and around in the list, then we'll move on to -actually reading emails. - -### The Pager - -Press `return` in the index to open the selected email. This view is called the -pager: - -![Mutt's Pager](/media/images{{ parent_url }}/mutt-pager.png) - -Like before, let's add a few settings: - - :::text - # Pager View Options --------------------------------- - set pager_index_lines = 10 # number of index lines to show - set pager_context = 3 # number of context lines to show - set pager_stop # don't go to next message automatically - set menu_scroll # scroll in menus - set tilde # show tildes like in vim - unset markers # no ugly plus signs - - set quote_regexp = "^( {0,4}[>|:#%]| {0,4}[a-z0-9]+[>|]+)+" - alternative_order text/plain text/enriched text/html - -This is a good, sane starting point. And now for a few extra key bindings: - - :::text - # Pager Key Bindings --------------------------------- - bind pager k previous-line - bind pager j next-line - bind pager gg top - bind pager G bottom - - bind pager R group-reply - - # View attachments properly. - bind attach view-mailcap - -The first few make scrolling behave like it does in the index. We're also going -to use the same key for reply all here. Consistency will make it easier to get -Mutt into your fingers. - -Don't worry about the last one -- that's to make sure Mutt treats attachments -properly. - -Go ahead and try reading some emails. Remember that `?` will always give you -a list of keys and their functions. - -### Attachments - -Now that we're all set for reading plain text email, it's time to deal with -attachments. - -When you're in the pager view reading an email with attachments, you can press -`v` to view a list of them: - -![Attachment List](/media/images{{ parent_url }}/mutt-attachments.png) - -Scroll through the list with `j` and `k` and press `return` to view one. But -first we need to tell Mutt how to view things that aren't text! - -For that we need to create a `~/.mutt/mailcap` file. Here's a sample to get you -started: - - :::text - # MS Word documents - application/msword; ~/.mutt/view_attachment.sh %s "-" '/Applications/TextEdit.app' - - # Images - image/jpg; ~/.mutt/view_attachment.sh %s jpg - image/jpeg; ~/.mutt/view_attachment.sh %s jpg - image/pjpeg; ~/.mutt/view_attachment.sh %s jpg - image/png; ~/.mutt/view_attachment.sh %s png - image/gif; ~/.mutt/view_attachment.sh %s gif - - # PDFs - application/pdf; ~/.mutt/view_attachment.sh %s pdf - - # HTML - text/html; ~/.mutt/view_attachment.sh %s html - - # Unidentified files - application/octet-stream; ~/.mutt/view_attachment.sh %s "-" - -The `view_attachment.sh` script is from [here][attachment]. Here's a link to -[my copy][attachment-mirror] in case that site ever goes down. Grab the script, -chmod it to executable, and stick it in `~/.mutt`. - -You can poke around and figure out how it works, or you can just not worry about -it and get on with life. I recommend the latter (at least for now). - -Now you can press `return` to open an attachment in the proper program. - -[attachment]: http://linsec.ca/Using_mutt_on_OS_X#mailcap -[attachment-mirror]: https://bitbucket.org/sjl/dotfiles/src/tip/mutt/view_attachment.sh - -### URLs - -One thing you'll probably want to do while reading email is open links. Many -terminal programs like iTerm2 let you command-click on a link to open it, but -this is Mutt! We shouldn't have to use the mouse! - -We're going to use a small helper program called urlview to make it easy to open -URLs in email. First, install it with `brew install urlview`. Then make -a `~/.urlview` file with the following contents: - - :::text - COMMAND open %s - -This tells urlview what command to use to open a URL. We're just going to use -the OS X `open` command to do the right thing. - -Next, add the following line to your `~/.muttrc`: - - :::text - macro pager \Cu "|urlview" "call urlview to open links" - -Now when you're reading an email with links in it you can press `Ctrl-u` to open -urlview. You'll see a screen like this: - -![urlview screen](/media/images{{ parent_url }}/mutt-urls.png) - -Navigate with `j`, `k`, `gg`, `G`, or `/` and press `return` when the desired -link is selected. That link will be filled in at the bottom of the screen in -case you want to edit it, and you can press `return` one more time to actually -open it in your default browser. - -That about wraps it up for reading email. Now it's time to write some! - -Writing Email -------------- - -Writing email is one of the best parts of Mutt. First let's add a few settings -to get things nice and sane: - - :::text - # Compose View Options ------------------------------- - set realname = "Steve Losh" # who am i? - set envelope_from # which from? - set sig_dashes # dashes before sig - set edit_headers # show headers when composing - set fast_reply # skip to compose when replying - set askcc # ask for CC: - set fcc_attach # save attachments with the body - unset mime_forward # forward attachments as part of body - set forward_format = "Fwd: %s" # format of subject when forwarding - set forward_decode # decode when forwarding - set attribution = "On %d, %n wrote:" # format of quoting header - set reply_to # reply to Reply to: field - set reverse_name # reply as whomever it was to - set include # include message in replies - set forward_quote # include message in forwards - -You can reply to an email with `r` in the index or pager, or start a fresh one -with `m`. - -There's actually not a lot to say about writing mail, because Mutt itself -doesn't handle it! Mutt passes control off to the text editor of your choice. -Just specify your editor in your `~/.muttrc`: - - :::text - set editor = "vim" # Use terminal Vim to compose email. - set editor = "mvim -f" # Use MacVim to compose email. - set editor = "subl -w" # Use Sublime Text 2 to compose email. - -Any command that takes a filename and doesn't return until you're done can be -used here. - -This is fantastic because it means you can use an editor you're already -comfortable and fast in to write email instead of learning yet another set of -shortcuts. - -Once you save the email in your editor and close it, Mutt will present you with -a menu that looks like this: - -![Sending Screen](/media/images{{ parent_url }}/mutt-send-1.png) - -You can press `e` to go back and edit the mail, `a` to add attachments, and so -on (the options are listed at the top of the screen). - -Before we can continue we need to tell Mutt how to send email. Press `q` to -discard the email for now. - -Sending Email -------------- - -Mutt does have (some) built-in SMTP support, but we're going to use a separate -program to do our sending for a few reasons. - -First, Mutt's SMTP support was considered "experimental" the last time -I checked. Sending email is kind of important, so we'll stick with something -tried and true. - -Second, we want a method that won't require our password in a plaintext config -file. - -Go ahead and install the `msmtp` program through Homebrew with `brew install -msmtp`. - -Next we're going to need to create a `~/.msmtprc` file with the following -contents: - - :::text - account stevelosh - host smtp.gmail.com - port 587 - protocol smtp - auth on - from steve@stevelosh.com - user steve@stevelosh.com - tls on - tls_trust_file ~/.mutt/Equifax_Secure_CA.cert - - account default : stevelosh - -`msmtp` will look in your keychain for your SMTP password, which we added -earlier. No plaintext passwords! - -The other "interesting" bit here is the `tls_trust_file`. We're going to be -connecting to Gmail's SMTP server over SSL, and `msmtp` needs to know if it can -trust the certificate that the server on the other end is sending back. - -Copy the following and paste it into the path `tls_trust_file` is set to: - - :::text - -----BEGIN CERTIFICATE----- - MIIDIDCCAomgAwIBAgIENd70zzANBgkqhkiG9w0BAQUFADBOMQswCQYDVQQGEwJVUzEQMA4GA1UE - ChMHRXF1aWZheDEtMCsGA1UECxMkRXF1aWZheCBTZWN1cmUgQ2VydGlmaWNhdGUgQXV0aG9yaXR5 - MB4XDTk4MDgyMjE2NDE1MVoXDTE4MDgyMjE2NDE1MVowTjELMAkGA1UEBhMCVVMxEDAOBgNVBAoT - B0VxdWlmYXgxLTArBgNVBAsTJEVxdWlmYXggU2VjdXJlIENlcnRpZmljYXRlIEF1dGhvcml0eTCB - nzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAwV2xWGcIYu6gmi0fCG2RFGiYCh7+2gRvE4RiIcPR - fM6fBeC4AfBONOziipUEZKzxa1NfBbPLZ4C/QgKO/t0BCezhABRP/PvwDN1Dulsr4R+AcJkVV5MW - 8Q+XarfCaCMczE1ZMKxRHjuvK9buY0V7xdlfUNLjUA86iOe/FP3gx7kCAwEAAaOCAQkwggEFMHAG - A1UdHwRpMGcwZaBjoGGkXzBdMQswCQYDVQQGEwJVUzEQMA4GA1UEChMHRXF1aWZheDEtMCsGA1UE - CxMkRXF1aWZheCBTZWN1cmUgQ2VydGlmaWNhdGUgQXV0aG9yaXR5MQ0wCwYDVQQDEwRDUkwxMBoG - A1UdEAQTMBGBDzIwMTgwODIyMTY0MTUxWjALBgNVHQ8EBAMCAQYwHwYDVR0jBBgwFoAUSOZo+SvS - spXXR9gjIBBPM5iQn9QwHQYDVR0OBBYEFEjmaPkr0rKV10fYIyAQTzOYkJ/UMAwGA1UdEwQFMAMB - Af8wGgYJKoZIhvZ9B0EABA0wCxsFVjMuMGMDAgbAMA0GCSqGSIb3DQEBBQUAA4GBAFjOKer89961 - zgK5F7WF0bnj4JXMJTENAKaSbn+2kmOeUJXRmm/kEd5jhW6Y7qj/WsjTVbJmcVfewCHrPSqnI0kB - BIZCe/zuf6IWUrVnZ9NA2zsmWLIodz2uFHdh1voqZiegDfqnc1zqcPGUIWVEX/r87yloqaKHee95 - 70+sB3c4 - -----END CERTIFICATE----- - -If you're paranoid and don't trust that I'm giving you the right cert (or that -someone has hacked my site and changed it), you can generate it yourself. I'll -leave that to you -- if you care enough about it you'll figure it out. - -Now we need to tell Mutt to use msmtp. Add the following to your `~/.muttrc` -file: - - :::text - set from = "steve@stevelosh.com" - set sendmail = "/usr/local/bin/msmtp -a stevelosh" - set sendmail_wait = 0 - unset record - -The `-a stevelosh` will need to change to whatever you named your account in the -msmtp config. - -The `unset record` line tells Mutt to not append a copy of every email you send -to a file on your hard drive. Gmail will save the emails you send in the sent -folder, so you'll get the the next time you sync with offlineimap anyway. - -The `sendmail_wait` line tells Mutt to wait for the msmtp program to finish -sending the mail before returning control, instead of running it in the -background. This makes it obvious if there's a problem sending a message, which -I prefer to silent, backgrounded failures. - -Now you can send email! Awesome! - -Once you've composed a test email and saved it you'll be presented with a screen -like this (which we saw in the previous section): - -![Sending Screen](/media/images{{ parent_url }}/mutt-send-1.png) - -The keys you need are listed along the top. Pressing `y` now will invoke msmtp -and send your email! - -You'll see "Sending message..." at the bottom of the screen while msmtp is -working. If there's a problem, Mutt will tell you the error. Figure it out -before moving on. - -Postponing Drafts ------------------ - -Sometimes I like to read and respond to email without an internet connection, -then actually send the replies when I get back to civilization. - -Since all of my email is stored locally, reading it offline is trivial. - -To reply or compose offline, I use Mutt's "postpone" feature. First I've added -the following line to my `~/.muttrc`: - - :::text - bind compose p postpone-message - -Now when I'm at the sending screen instead of pressing `y` to send I can press -`p` to postpone the message. This places the message in the drafts folder. - -The drafts folder in Mutt is just a normal folder of emails like any other. -When you sync with offlineimap your postponed email will get pushed up to Gmail -as a draft. You can edit it in the Gmail web interface if you like, and those -edits will sync back down too. - -If you want to edit your drafts (postponed messages) locally, you need to -"recall" them. I have the following line in my `~/.muttrc` to set the key to -the same one as I used to postpone it in the first place: - - :::text - bind index p recall-message - -You can also use `m` to start writing a new email, and mutt will prompt you if -there are existing postponed messages. - -Once you hit `p` (or `m` and then select "yes") Mutt will show you a list of the -postponed messages. Select one and press `return` to start editing it again. -If there's only one it will skip this step and simply open it for you -immediately. - -Once you're done editing you can either postpone it again or send it as usual. - -Unfortunately you have to go through the "recall → edit → send" process each -time. As far as I know there's no way to simply run down a list of postponed -emails sending each one with a single keystroke. But that's not *too* bad, and -it's great to be able to work offline like this. - -Once you send the postponed email (through Mutt or the Gmail web interface) it -disappears from the drafts folder and the postponed list as you would expect. - -Contacts --------- - -Next we'll want to get Mutt to autocomplete our contacts from the OS X address -book. I like using the OS X address book because it automatically syncs between -my laptops and phone, so I only need to maintain one address list. - -### Autocompleting - -Unfortunately I've got some bad news for you: you're going to need to install -XCode. - -No, not the command-line developer tools. The full XCode. I'm sorry, but trust -me when I say it's going to save you a lot of pain, so just grumble to yourself -a bit and do it. - -Okay, now that you've got XCode you can install the `contacts` program through -Homebrew with `brew install contacts`. - -`contacts` is a command-line program that you can use to query your address -book. To tell Mutt how to use it add the following lines to your -`~/.mutt/muttrc`: - - :::text - set query_command = "contacts -Sf '%eTOKEN%n' '%s' | sed -e 's/TOKEN/\t/g'" - bind editor complete-query - bind editor ^T complete - -Now when you're filling out an email address field you can type a few characters -and hit Tab to get a screen like this: - -![Contacts](/media/images{{ parent_url }}/mutt-contacts-1.png) - -You can use `j` and `k` to select an item, press return to complete it. Press -`q` if you've changed your mind and want to cancel the completion. Look at the -top of the screen for more handy little keys you can use here. - -If there's only one item in the list Mutt won't bother showing you this screen -and will just complete it right away. - -This completion searches more than just the email address. It'll also search -the names and possibly other fields from the address book entries as well. - -### Adding Contacts - -What about adding contacts to your address book? Any contacts you add on your -phone will automatically be synced, but what if you're reading your mail in Mutt -and just want to add the sender as a contact without leaving your command line? - -For this I use a little script that [Simone Manganelli][simx] wrote for me -called `addcontact`. You can [get it here][addcontact] and stick it in your -`$PATH` somewhere. It's just a command-line utility that you can use like this: - - :::text - $ addcontact Steve Losh steve@stevelosh.com - $ addcontact "Steve Losh" steve@stevelosh.com - $ addcontact Steve Losh work steve@pculture.org - -As you can see, it's pretty flexible. - -**Note:** This utility always adds a new contact record, so if you add someone -that's already in there you're going to get a duplicate entry. If that happens -you can search for the entries in Contacts.app, select the duplicates, and use -"Card → Merge Selected Cards" to combine them. - -Okay, so we can now add contacts from the command line. You could set up a Mutt -macro to automatically add senders without too much trouble, but I don't do -that. If I want to add a contact I just hit `!` and type out the shell command. -It's not that much work, and sometimes the name in the "From:" field is in -a weird format like "Last, First Middle" instead of "First Last", so this gives -me a chance to clean it up before I add it. - -[simx]: http://simx.me/ -[addcontact]: https://bitbucket.org/sjl/dotfiles/src/tip/bin/addcontact - -Searching Email ---------------- - -If you have more than a screen full of email in any given folder, you're going -to want a way to search through it. Mutt's built-in searching is a good start, -but I also use another program to get a bit more power. - -### Vanilla Searching - -There are two main ways to search your email in Mutt: plain searching and -"limiting". - -Plain searching is done with the `/` key. It works similarly to Vim or less: -you press `/`, type your query, and press enter to perform the search. You can -use `n` to move to the next match, and the search will loop around to the top if -it hits the bottom. - -I have the following lines in my `~/.muttrc` to bind `N` to go to the *previous* -match: - - :::text - bind index N search-opposite - bind pager N search-opposite - -Normally the `N` key marks a message as unread (or "new"). I personally never -want to do that. Unread mail should be "mail that has not been read". If you -use that feature you'll want to rebind it to something else. - -Two things about queries: - -1. They are regular expressions (actually it's more powerful than than, see [the - documentation][patterns] for more information). -2. They only search the To and Subject fields (*not* the message bodies!). - -I generally use this kind of searching when I see the email I want to open on -the screen but don't feel like pressing `j` or `k` forty times to move through -the list. When I'm actually trying to find a message I can't already see on the -screen I use limiting. - -[patterns]: http://www.mutt.org/doc/manual/manual-4.html#ss4.2 - -### Vanilla Limiting - -Limiting is the other way Mutt provides for searching mail. It's done with the -`l` key by default. - -Like `/`, `l` will ask you for a pattern. But instead of simply moving you to -the next message that matches the pattern, Mutt will *hide* messages that don't -match. - -This lets you see all the ones that match in a single list. This list works -just like a normal list. You can search it with `/`, tag things, and so on as -you normally would. It's really quite nice once you get used to it. - -To remove the limiting once you're done, you can limit to the special value -`all`. I've added a line to my `~/.muttrc` so I can do that with a single key: - - :::text - macro index a "all\n" "show all messages (undo limit)" - -**Note:** This shadows the `create-alias` function which I never use. - -Limit queries work exactly like search queries, so you can do powerful stuff -like `~f arthur ~C honza ~s api` ("limit to messages from 'arthur', to or cced -to 'honza', containing 'api' in the subject"). - -### Full-Text Searching - -By now you're probably wondering how to search the full text of messages. There -are two ways: one simple and slow, the other complicated and fast. - -First is the simple, slow way. You can use `~B` in your searches and limits to -search inside the entire message. If your folder only has a hundred messages -this works great. But once you have a few more (my archive has about 30,000 at -the moment, and I prune it fairly often) it quickly becomes too slow to be -usable. - -I use a program called [notmuch][] to index and search my email. It's blazingly -fast and works pretty well. - -First, install it with `brew install notmuch`. Now you need -a `~/.notmuch-config` file. Run `notmuch setup` to generate one. It's pretty -straightforward. When it asks you for the path to your archive, that's the path -to the folder containing all your individual IMAP folders -(`~/.mail/steve-stevelosh.com` in my setup). - -Now run `notmuch new` to perform the initial index. It might take a while if -you have a lot of folders. Subsequent indexes will be much faster since they'll -only reindex whatever changed since the last time. - -You'll want to reindex when your email changes, so add a hook in your -`~/.offlineimaprc` to run `notmuch new` whenever offlineimap syncs your email: - - :::text - [Account SteveLosh] - localrepository = SteveLosh-Local - remoterepository = SteveLosh-Remote - status_backend = sqlite - postsynchook = notmuch new - -Now you can use `notmuch search foo` to search your mail for "foo" from the -command line. Try a couple of queries and make sure it works. - -Now we need to hook notmuch into Mutt. There are a number of different ways to -do this, all of them hacky and ugly. I'll describe how I do it. - -A quick overview of how this is going to work: - -1. You'll press a key in Mutt to activate searching. -2. You'll type your query and press return. -3. Your mail will be searched with `notmuch`. -4. Τhe resulting messages will be symlinked into a temporary maildir folder. -5. That temporary folder will be opened in Mutt. - -First get [mutt-notmuch-py][]. The original `mutt-notmuch` is a Perl script -with many external requirements that are a pain to install, and it doesn't work -on OS X. `mutt-notmuch-py` is a Python script with zero external requirements. - -`mutt-notmuch-py` is going to handle steps 2 through 4 in the list. Get the -script into your `$PATH` somehow and then run it: - - :::text - $ mutt-notmuch-py -G ~/.mail/temporary/search - Query: foo - - $ ls ~/.mail/temporary/search - cur new - -The `-G` tells it to not perform certain Gmail-specific stuff that we don't -need. The path is where it will create the temporary maildir folder with the -results. Each time you run it this folder will be wiped clean before the new -results are linked into it. - -Now we need to handle the first and last points in the list. I have the -following mapping in my `~/.muttrc`: - - :::text - macro index S "unset wait_keymutt-notmuch-py ~/.mail/temporary/search+temporary/search" "search mail (using notmuch)" - -That's a lot to take in, so let's see how it works piece by piece: - - :::text - macro index S - -We're going to use the `S` key to perform a full search of all of our mail. - - :::text - unset wait_key - -Unset the `wait_key` Mutt option to prevent Mutt from asking us to press a key -after the search has finished. - - :::text - mutt-notmuch-py -G ~/.mail/temporary/search - -Run `mutt-notmuch-py`. Control will pass over to that script and it will ask -you for your query, run the search, set up the results folder, and then hand -control back to Mutt. - - :::text - +temporary/search - -Now we change over to the temporary folder in Mutt, and we're now looking at -a list of search results! Since this is a real Maildir folder like any other -one we can use the full range of tools to interact with it (searching, limiting, -etc). - -Finally, let's get this search results folder in our sidebar so we can see where -we are visually at all times: - - :::text - mailboxes +steve-stevelosh.com/INBOX \ - +steve-stevelosh.com/vim \ - +steve-stevelosh.com/clojure \ - ... - +temporary/search \ - -Now the search results folder can be navigated like any other one. That's it -for email searching! Now you should have a setup that you can use in real life -to manage your email. - -[notmuch]: http://notmuchmail.org/ -[mutt-notmuch-py]: https://github.com/honza/mutt-notmuch-py - -Conclusion ----------- - -Mutt is definitely a beast. It's old, crufty, and ugly, but if you spend the -time to set it up and learn to use it you'll be rewarded with a fast, powerful, -customizable environment for working with your email. - -A wonderful trend these days is that more and more sites are including the -ability to respond to comments and such by simply replying to their notification -emails. This means that often you can reply to Facebook emails, comment on -GitHub pull requests, and respond to Bitbucket issues all without leaving the -comfort of your finely-tuned email client. - -Mutt's not for everyone, but if you deal with a lot of email and have some time -to spend you should give it a try. You just might learn to love the old dog. - -{% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/10/the-homely-mutt.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2012/10/the-homely-mutt.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,1360 @@ ++++ +title = "The Homely Mutt" +snip = "Sparrow's dead? Why not try Mutt?" +date = 2012-10-01T10:30:00Z +draft = false + ++++ + +Now that [Sparrow][] is [effectively dead][sparrow-dead] many of its users will +be looking for a new email client. If you're not afraid of the terminal you may +want to give [Mutt][] a try. + +Mutt certainly isn't the prettiest email client around, and its +setup/configuration process is one of the ugliest out there. But once you get +it set up it's got a lot of advantages over many other email clients. + +In this post I'll show you how to set up Mutt on OS X like I do. + +[Sparrow]: http://sparrowmailapp.com/ +[sparrow-dead]: http://www.theverge.com/2012/7/20/3172222/google-buys-sparrow-mail +[Mutt]: http://www.mutt.org/ + +{{% toc %}} + +How I Use Email +--------------- + +This setup is going to be specific to the way I work with email. Notably: + +* I have a Google Apps account that provides my steve@stevelosh.com email + address. +* I have many other email addresses, but they all simply forward to my main one. +* All mail I send comes from steve@stevelosh.com. +* I store my contacts in the OS X address book. +* All email comes into my inbox (or to a folder for a specific mailing list). +* Once I'm done with an email, I remove it from my inbox and it lives in the + "All Mail" archive. I don't sort email into folders after it arrives. +* I sometimes read email offline and mark it for deletion, then sync that + deletion back to the server once I get online again. +* Sometimes I write email without an internet connection and send it once I get + connected again. + +My email setup is tailored around those requirements, so that's what it does +best. Mutt is very configurable though, so if you work differently you can +probably bend it to make it do what you want. + +In particular, extending this setup to work with multiple email accounts +wouldn't be too much trouble. I used to work with two separate accounts until +I said "screw it, I'll just use the one". + +Other Guides and Resources +-------------------------- + +I've used a lot of other guides to figure out how to get this giant Rube +Goldberg machine of an email client working. Here are a few of them: + +* +* +* +* +* +* +* +* +* +* + +Overview +-------- + +I'm going to give it to you straight: getting this whole contraption set up is +going to take at least an hour from start to finish, not counting the time it'll +take to download all of your email and install stuff. Set aside an evening if +you're serious about this. + +It's an investment, and you might not want to make it. If not, go use +Thunderbird, er, Sparrow, er, I don't know, the Gmail web interface or +something. + +Mutt on its own doesn't do very much, so we're going to combine it with a few +other things to get the job done. Here's a bird's eye view of what it'll look +like when we're done: + +![Diagram](/media/images/blog/2012/10/what-the-mutt.png) + +If this diagram doesn't make you run screaming you might just be masochistic +enough to make it through the initial setup of Mutt. If you do, you'll be +rewarded with email bliss that won't go away when Google or Facebook decide to +toss some money around. + +Getting Email +------------- + +First thing's first: we're going to pull down our email from Gmail to our local +machine. All of it. It'll take a while the first time you sync, but has a few +benefits. + +### Why Local Email? + +Having a local copy of all of your email means you've always got access to it, +no matter where you are. Looking for that one person's address they emailed you +six years ago when you're trying to find their house and you don't have an +internet connection? No problem, it's on your hard drive. + +This also acts as a backup in case Google ever decides to kill your Gmail +account. It'll be stored in a common format that a lot of programs can read, so +you've got a safety net. And the email is stored as normal files, so if you use +something like Time Machine or [Backblaze][] that's yet another free backup. + +In this setup **all of your email is stored as plain text**. If you want it +encrypted just use OS X's full-disk encryption and you're set. + +I use [offlineimap][] to pull email down from Gmail and get it on my hard drive. +Offlineimap will also sync any changes you make to this local copy back up to +Gmail. + +[offlineimap]: http://offlineimap.org/ +[Backblaze]: http://www.backblaze.com/partner/af3574 + +### The Alternative + +You may not care as much about reading your email offline as I do. If you can +tolerate always needing an internet connection to read your mail, you can skip +this painful section and follow [this guide][roma] instead. + +You'll probably still find the other sections of this post interesting though. + +[roma]: http://empt1e.blogspot.com/2009/10/using-mutt-with-gmail-imap-complete.html + +### Installing offlineimap + +I've gone through a number of laptops in the past few years, and each time +I spend a painful half hour or so screwing around with the latest version of +offlineimap's backwards-incompatible changes. + +If you're determined to run the latest version of offlineimap, you can install +it with pip or something. If you just want to download your fucking email and +get on with your life, you can follow the instructions I've laid out for you +here: + +```text +git clone git://github.com/spaetz/offlineimap.git +cd offlineimap +git checkout 679c491c56c981961e18aa43b31955900491d7a3 +python setup.py install +``` + +That's the version I'm using. It works. You can use a newer one if you want, +but expect to spend some time figuring out how to fix the configuration in this +post to work with whatever breaking changes have been made since then. The last +time I tried this I got to rewrite all my nametrans stuff. That was fun. + +### Configuring offlineimap + +Once you've got offlineimap installed, you'll need to create +a `~/.offlineimaprc` file. You can keep it in your dotfiles repo and symlink it +into place if you want. Here's a sample to get you started: + +```text +[general] +ui = TTY.TTYUI +accounts = SteveLosh +pythonfile=~/.mutt/offlineimap.py +fsync = False + +[Account SteveLosh] +localrepository = SteveLosh-Local +remoterepository = SteveLosh-Remote +status_backend = sqlite +postsynchook = notmuch new + +[Repository SteveLosh-Local] +type = Maildir +localfolders = ~/.mail/steve-stevelosh.com +nametrans = lambda folder: {'drafts': '[Gmail]/Drafts', + 'sent': '[Gmail]/Sent Mail', + 'flagged': '[Gmail]/Starred', + 'trash': '[Gmail]/Trash', + 'archive': '[Gmail]/All Mail', + }.get(folder, folder) + +[Repository SteveLosh-Remote] +maxconnections = 1 +type = Gmail +remoteuser = steve@stevelosh.com +remotepasseval = get_keychain_pass(account="steve@stevelosh.com", server="imap.gmail.com") +realdelete = no +nametrans = lambda folder: {'[Gmail]/Drafts': 'drafts', + '[Gmail]/Sent Mail': 'sent', + '[Gmail]/Starred': 'flagged', + '[Gmail]/Trash': 'trash', + '[Gmail]/All Mail': 'archive', + }.get(folder, folder) +folderfilter = lambda folder: folder not in ['[Gmail]/Trash', + 'Nagios', + 'Django', + 'Flask', + '[Gmail]/Important', + '[Gmail]/Spam', + ] +``` + +It's kind of long, so let's go through it line by line and see what's going on. + +```text +[general] +ui = TTY.TTYUI +accounts = SteveLosh +pythonfile=~/.mutt/offlineimap.py +fsync = False +``` + +First we tell offlineimap to use the `TTY.TTYUI` ui. Yes, this program that +syncs your email has multiple user interfaces. I guess if you can't decide what +color the bikeshed should be you can just build a whole bunch of bikesheds +instead. + +Then we specify the accounts. There's only one because as I said before: +I only use a single email account that all my addresses forward to. If you +wanted to have many, you'd change this line. + +The `pythonfile` is just a file that offlineimap will parse (as Python) before +loading the rest of the config, so you can define custom helper functions more +easily. We'll see more of this later. + +We're also telling offlineimap that it doesn't need to fsync after every single +operation. This will speed things up, and since it's just a local copy it's +typically not a big deal if we lose an email here and there from a crash (it'll +just be synced the next time anyway). + +```text +[Account SteveLosh] +localrepository = SteveLosh-Local +remoterepository = SteveLosh-Remote +status_backend = sqlite +``` + +This next section hooks up a few things. First, it tells offlineimap which +local and remote repositories to use for the account. Manual configuration +instead of sane defaults is a recurring theme we'll see throughout this process. + +Hey, I titled the entry "The *Homely* Mutt" for a reason. + +We're also going to use a SQLite-based cache for this account. If you don't +already have SQLite you'll want to get it with `brew install sqlite`. + +```text +[Repository SteveLosh-Local] +type = Maildir +localfolders = ~/.mail/steve-stevelosh.com +nametrans = lambda folder: {'drafts': '[Gmail]/Drafts', + 'sent': '[Gmail]/Sent Mail', + 'flagged': '[Gmail]/Starred', + 'trash': '[Gmail]/Trash', + 'archive': '[Gmail]/All Mail', + }.get(folder, folder) +``` + +Now we're getting to the meat of the configuration. This "local repository" is +going to be the mail as it sits on our hard drive. We're going to use the +[Maildir format][maildir] because it plays nicely with Mutt (and tons of other +stuff). + +Then we specify the path where we're going to keep the mail. This is going to +take a lot of space if you've got a lot of mail. Attachments are downloaded +too. When I said you're getting an offline copy of all your email I meant *all* +of it. + +I think offlineimap needs the `~/.mail` directory created for it. It's been +a while since I did this, so I might be wrong, but if it complains about not +being able to access the mail folders just go ahead and `mkdir ~/.mail`. + +Next we have the craziest part of the configuration: name translation. + +Here's the issue: offlineimap needs to know how to translate the names of +folders on the IMAP server to folder names on your hard drive. + +Also, Gmail doesn't actually use *folders* but its own concept called "labels". +But since the IMAP protocol doesn't know about labels, it fakes them by making +them appear to be folders. + +User-created labels in Gmail (like "Mercurial" or "Clients") will appear as +folders with those names through IMAP. + +Built-in, special Gmail folders have names that start with `[Gmail]/`. We need +to turn those into something sane for our hard drive, so that's what this +nametrans setting is for. It's a Python function that takes the remote folder +name and returns the name that should be used on your local hard drive. + +Yes, you read that right. This is Python code embedded in the right hand side +of an INI file's setting assignment. I am not fucking with you, this is +seriously how you do it. Go ahead and crack open that beer now. + +So the "Sent Mail" folder in your Gmail account will be synced to +`~/.mail/steve-stevelosh.com/sent`. Cool. + +(No, I don't know what would happen if you created a label called `[Gmail]/All +Mail` in Gmail. If you try please let me know, but I take no responsibility if +it ends with all your email being deleted.) + +[maildir]: https://en.wikipedia.org/wiki/Maildir + +```text +[Repository SteveLosh-Remote] +maxconnections = 1 +type = Gmail +remoteuser = steve@stevelosh.com +remotepasseval = get_keychain_pass(account="steve@stevelosh.com", server="imap.gmail.com") +realdelete = no +nametrans = lambda folder: {'[Gmail]/Drafts': 'drafts', + '[Gmail]/Sent Mail': 'sent', + '[Gmail]/Starred': 'flagged', + '[Gmail]/Trash': 'trash', + '[Gmail]/All Mail': 'archive', + }.get(folder, folder) +folderfilter = lambda folder: folder not in ['[Gmail]/Trash', + 'Nagios', + 'Django', + 'Flask', + '[Gmail]/Important', + '[Gmail]/Spam', + ] +``` + +Finally, the home stretch. The last section described the folder on our local +hard drive, and this one describes our Gmail account. + +First, we tell offlineimap to only ever use a single connection at a time. You +can try increasing this number for better performance, but in my experience +Google is not afraid to enforce its rate limits and would cut me off fairly +often when I tried that. Just leave it at one if you want to be safe. + +Next is the type. Luckily offlineimap provides a `Gmail` type that handles +a lot of the craziness that is Gmail's IMAP setup. Nice. + +Then we have the username. Nothing special here, except that if you have +a non-apps account (i.e.: an actual vanilla Gmail account) you may or may not +need to include the `@gmail.com` in the username. I don't know. If one doesn't +work, just try the other. + +Next we have `remotepasseval`. This is a bit of Python code (drink!) that +should return the password for the account. + +What is this `get_keychain_pass` function? Well, remember when we saw the +`pythonfile` setting back in the general section? It's a function defined in +there. I'll talk about that in the next section, for now just accept that it +works. + +Next we set `realdelete` to no. If this is set to yes, then deleting an email +in your inbox would actually delete it entirely. When you set it to no, then +deleting an email from your inbox (or any label's folder) will leave it in +Gmail's All Mail. + +If you want to really delete an email, you'll need to delete it from All Mail +(which is named archive on our local filesystem, remember?). I feel like this +is a good compromise. I rarely care about actually deleting mail, given that +I have many unused gigabytes available on Gmail. + +Next we have another nametrans setting. This is a Python function (drink!) just +like the one for the local repository, except it goes in the other direction. +It takes the name of a local folder and returns the name of the folder on the +IMAP server. Knowing this, it should be easy to understand this setting. + +Finally, we have `folderfilter`. This is a Python function (drink!) that takes +a **remote** folder name and returns `True` if that folder should be synced, or +`False` if it should **not** be synced. I've chosen to skip syncing my Spam and +Trash folders, as well as a few mailing list labels I don't check all that +often. Customize this to your own taste. + +### Retrieving Passwords + +We're almost ready, but there's one more thing we need to do, and that's +implement a secure way for offlineimap to get access to our Gmail password. + +If you don't care too much about security, you *can* configure offlineimap with +a plaintext password right in the config file. But don't do that. It'll only +take a minute to do this securely. + +First, you need to add your Gmail password into your OS X keychain. Open the +Keychain Access app and press the `+` button: + +![Keychain 1](/media/images/blog/2012/10/keychain-1.png) + +Then fill out the form. The "Keychain Item Name" should be +`http://imap.gmail.com`. The "Account Name" should be your email address. The +password should be your password: + +![Keychain 2](/media/images/blog/2012/10/keychain-2.png) + +Press "Add". Now repeat the process for the SMTP server. The "Keychain Item +Name" should be `smtp://smtp.gmail.com`. The "Account Name" should be your +email address. The password should be your password: + +![Keychain 3](/media/images/blog/2012/10/keychain-3.png) + +Now we need to create the `offlineimap.py` file we pointed offlineimap to +earlier. It needs to contain the `get_keychain_pass` function, which takes an +`account` and `server` and return the password. Here's the file I'm using: + +```python +#!/usr/bin/python +import re, subprocess +def get_keychain_pass(account=None, server=None): + params = { + 'security': '/usr/bin/security', + 'command': 'find-internet-password', + 'account': account, + 'server': server, + 'keychain': '/Users/sjl/Library/Keychains/login.keychain', + } + command = "sudo -u sjl %(security)s -v %(command)s -g -a %(account)s -s %(server)s %(keychain)s" % params + output = subprocess.check_output(command, shell=True, stderr=subprocess.STDOUT) + outtext = [l for l in output.splitlines() + if l.startswith('password: ')][0] + + return re.match(r'password: "(.*)"', outtext).group(1) +``` + +In a nutshell, it uses `/usr/bin/security` to retrieve the password. Read +through the code if you're curious. + +This is not *completely* secure, but it's better than having your password in +a plaintext file in your home directory. + +Whew! Time to actually run this thing and pull down our email! + +### Running offlineimap + +Assuming everything is in place, open a terminal and run offlineimap: + + offlineimap + +Go read a book, because this is going to pull down all the email (with +attachments) in any folders you didn't exclude in the config file. + +**If there's an error, stop and figure out what went wrong**. Remember, +offlineimap is a *two-way* sync, so there's always the possibility it'll eat +your email if you seriously mess something up! I wish it had a +`--dont-touch-remote` option you could use as a safety net for the original +sync, but it doesn't, so be careful! + +In the future you can use `offlineimap -q` to run it in "quick mode". It'll +perform fewer checks but will generally be much faster. + +If you want to set up offlineimap to run every 5 minutes or so, you can use +`launchd`. `cron` does not work for some reason. I'm not entirely sure why. + +Personally I actually *like* having to press a key to fetch new mail. It's less +of a distraction than having new mail rolling in all the time. I can get new +email when I'm ready to actually look at it, rather than having it nagging me +all the time. + +The great part about offlineimap is that once you've got it configured and it's +successfully run once, it's pretty much rock solid from then on. You can run it +often, `Ctrl-c` it, put the laptop to sleep in the middle of a run, or `kill -9` +it, and it still won't lose emails. On the next sync it'll fix anything that's +missing. + +Mutt! +----- + +Now that you've got your email on your computer, it's finally time to start +using Mutt itself! + +### Installing + +Mutt can be installed in a bunch of different ways, but the easiest is through +Homebrew: + +```text +brew install mutt --sidebar-patch +``` + +The sidebar patch is a third-party patch that adds a sidebar to Mutt. I don't +know why it's not in core Mutt because it's insanely useful. Oh well, at least +Homebrew makes it simple to get. + +That's pretty much it for installation, but don't get too relaxed because you're +far from done. + +### Configuring + +Mutt is *very* configurable. This is great once you've become a power user and +want to mold it to your will, but terrible when you're just getting started. + +Mutt settings are kept in a `~/.muttrc` file. If this file doesn't exist Mutt +will look for `~/.mutt/muttrc` (note the lack of a dot in the filename), so you +can put it there if you prefer. + +Let's start by creating a basic `~/.muttrc` piece by piece (a lot of this was +taken from [this article][pris]). + +Once you've got a bit of Mutt under your belt you'll want to read [the +documentation][muttdoc] for these settings, but for now just use them to keep +things sane. + +[pris]: http://pbrisbin.com/posts/two_accounts_in_mutt +[muttdoc]: http://www.mutt.org/doc/manual/manual-6.html + +```text +# Paths ---------------------------------------------- +set folder = ~/.mail # mailbox location +set alias_file = ~/.mutt/alias # where to store aliases +set header_cache = ~/.mutt/cache/headers # where to store headers +set message_cachedir = ~/.mutt/cache/bodies # where to store bodies +set certificate_file = ~/.mutt/certificates # where to store certs +set mailcap_path = ~/.mutt/mailcap # entries for filetypes +set tmpdir = ~/.mutt/temp # where to keep temp files +set signature = ~/.mutt/sig # my signature file +``` + +Here we tell Mutt where to find the various folders it needs. + +```text +# Basic Options -------------------------------------- +set wait_key = no # shut up, mutt +set mbox_type = Maildir # mailbox type +set timeout = 3 # idle time before scanning +set mail_check = 0 # minimum time between scans +unset move # gmail does that +set delete # don't ask, just do +unset confirmappend # don't ask, just do! +set quit # don't ask, just do!! +unset mark_old # read/new is good enough for me +set beep_new # bell on new mails +set pipe_decode # strip headers and eval mimes when piping +set thorough_search # strip headers and eval mimes before searching +``` + +These are some basic options to make Mutt behave a bit more sanely. + +```text +# Sidebar Patch -------------------------------------- +set sidebar_delim = ' │' +set sidebar_visible = yes +set sidebar_width = 24 +color sidebar_new color221 color233 +``` + +These options are specific to the sidebar patch. + +```text +# Status Bar ----------------------------------------- +set status_chars = " *%A" +set status_format = "───[ Folder: %f ]───[%r%m messages%?n? (%n new)?%?d? (%d to delete)?%?t? (%t tagged)? ]───%>─%?p?( %p postponed )?───" +``` + +This gives us a pretty status bar with the information we care about (and none +of the stuff we don't). + +```text +# Header Options ------------------------------------- +ignore * # ignore all headers +unignore from: to: cc: date: subject: # show only these +unhdr_order * # some distros order things by default +hdr_order from: to: cc: date: subject: # and in this order +``` + +These options hide some of the extra email headers we don't care about when +viewing and composing email. + +Now it's time to fill on our account details: + +```text +# Account Settings ----------------------------------- + +# Default inbox. +set spoolfile = "+steve-stevelosh.com/INBOX" + +# Alternate email addresses. +alternates sjl@pculture.org still\.?life@gmail.com steve@ladyluckblues.com steve@pculture.org + +# Mailboxes to show in the sidebar. +mailboxes +steve-stevelosh.com/INBOX \ + +steve-stevelosh.com/vim \ + +steve-stevelosh.com/clojure \ + +steve-stevelosh.com/python \ + +steve-stevelosh.com/mercurial \ + +steve-stevelosh.com/archive \ + +steve-stevelosh.com/sent \ + +steve-stevelosh.com/drafts \ + +# Other special folders. +set mbox = "+steve-stevelosh.com/archive" +set postponed = "+steve-stevelosh.com/drafts" +``` + +Most of those should be self-explanatory. Fill in the appropriate values for +your mail folder(s). + +We'll add more as we go through the next few sections, but that's enough to get +us started. + +### Running + +Now that you've got Mutt configured you can run it: + +```sh +mutt +``` + +I like to always be in my `~/Desktop` folder when in Mutt, so that when I save +emails or attachments they go there by default. I have a little shell function +set up that `cd`s there for me before running Mutt: + +```sh +alias mutt 'cd ~/Desktop && mutt' +``` + +If you run the [new fish shell][fish], this is going to cause problems later +(long story, but it's related to the `read` builtin). Do yourself a favor and +head those confusing issues off at the pass with a fish function: + +```text +function mutt + bash --login -c 'cd ~/Desktop; /usr/local/bin/mutt' $argv; +end +``` + +Remember that if you use another shell like this you'll want to set up any +aliases and your `PATH` for that shell properly (probably identically to your +main shell). + +[fish]: http://ridiculousfish.com/shell/ + +Reading Email +------------- + +Once you start Mutt you should be looking at a list of the email in your inbox. +If so: congratulations! If not: stop and figure out what went wrong. + +### The Index + +When viewing a folder, Mutt presents you with a list of your email. This view +is called the "index": + +![Mutt's Index](/media/images/blog/2012/10/mutt-index.png) + +This entry isn't meant be a guide to setting up Mutt on OS X. For a full guide +on how to *use* Mutt, you can Google around for some tutorials, or just learn as +you go with `?`. The `?` key will show you a list of all the keys you can use +wherever you currently are, and what they do. + +Let's add a few lines to our `~/.muttrc` to make the index view behave a bit +more nicely: + +```text +# Index View Options --------------------------------- +set date_format = "%m/%d" +set index_format = "[%Z] %D %-20.20F %s" +set sort = threads # like gmail +set sort_aux = reverse-last-date-received # like gmail +set uncollapse_jump # don't collapse on an unread message +set sort_re # thread based on regex +set reply_regexp = "^(([Rr][Ee]?(\[[0-9]+\])?: *)?(\[[^]]+\] *)?)*" +``` + +I won't go into what those do here. You can read the documentation if you're +curious. + +Quit and rerun Mutt to see your changes. Mutt is a very lightweight program so +this should be fast. + +Let's also add a few key bindings in the index to make it easier to use: + +```text +# Index Key Bindings --------------------------------- +bind index gg first-entry +bind index G last-entry + +bind index R group-reply +bind index sync-mailbox +bind index collapse-thread + +# Ctrl-R to mark all as read +macro index \Cr "T~UN." "mark all messages as read" + +# Sync email +macro index O "offlineimap" "run offlineimap to sync all mail" +macro index o "offlineimap -qf INBOX" "run offlineimap to sync inbox" + +# Saner copy/move dialogs +macro index C "?" "copy a message to a mailbox" +macro index M "?" "move a message to a mailbox" +``` + +Remember to quit and rerun Mutt for them to take effect. + +We're going to use `j` and `k` to move around, so we may as well support Vim +keys like `gg` and `G` too. We'll use `R` for reply all, since that comes in +handy fairly often. `Ctrl-R` will mark all messages in the current folder as +read. + +Don't worry if you don't understand how all these bindings and macros work right +now. You can read the documentation later. + +The `tab` key is going to "commit" changes we've made in Mutt (like deleting an +email) to our local Maildir folder. Once those changes are in the Maildir +folder offlineimap will sync them to the server the next time it runs. This is +nice because it lets us recover if we accidentally do something stupid like +deleting the wrong email. + +**Note:** Mutt will also sync changes for a folder when you switch to +a different folder, and when you quit Mutt, so be aware of those. + +The `space` key will toggle collapsing of threads, which can be convenient when +viewing mailing lists (or any conversations with many messages). + +The `o` and `O` keys will run offlineimap to sync mail. Like I said before, +I prefer having to press a button to grab mail instead of it constantly grabbing +and nagging me. `o` will sync only the inbox (fast), and `O` will sync +everything (much slower). + +Finally we rebind `C` and `M` to perform the same operations they usually do, +but in a more user-friendly manner. + +While we're at it, let's add a way to navigate around the sidebar so we can +switch folders: + +```text +# Sidebar Navigation --------------------------------- +bind index,pager sidebar-next +bind index,pager sidebar-prev +bind index,pager sidebar-open +``` + +We're binding the `up` and `down` arrow keys to switch between folders, and +`right` to "enter" a folder. Give it a try. + +We don't need the arrows because we can navigate with `j` and `k`, but if you +prefer to rebind them to something else feel free. + +Practice moving between folders and around in the list, then we'll move on to +actually reading emails. + +### The Pager + +Press `return` in the index to open the selected email. This view is called the +pager: + +![Mutt's Pager](/media/images/blog/2012/10/mutt-pager.png) + +Like before, let's add a few settings: + +```text +# Pager View Options --------------------------------- +set pager_index_lines = 10 # number of index lines to show +set pager_context = 3 # number of context lines to show +set pager_stop # don't go to next message automatically +set menu_scroll # scroll in menus +set tilde # show tildes like in vim +unset markers # no ugly plus signs + +set quote_regexp = "^( {0,4}[>|:#%]| {0,4}[a-z0-9]+[>|]+)+" +alternative_order text/plain text/enriched text/html +``` + +This is a good, sane starting point. And now for a few extra key bindings: + +```text +# Pager Key Bindings --------------------------------- +bind pager k previous-line +bind pager j next-line +bind pager gg top +bind pager G bottom + +bind pager R group-reply + +# View attachments properly. +bind attach view-mailcap +``` + +The first few make scrolling behave like it does in the index. We're also going +to use the same key for reply all here. Consistency will make it easier to get +Mutt into your fingers. + +Don't worry about the last one — that's to make sure Mutt treats attachments +properly. + +Go ahead and try reading some emails. Remember that `?` will always give you +a list of keys and their functions. + +### Attachments + +Now that we're all set for reading plain text email, it's time to deal with +attachments. + +When you're in the pager view reading an email with attachments, you can press +`v` to view a list of them: + +![Attachment List](/media/images/blog/2012/10/mutt-attachments.png) + +Scroll through the list with `j` and `k` and press `return` to view one. But +first we need to tell Mutt how to view things that aren't text! + +For that we need to create a `~/.mutt/mailcap` file. Here's a sample to get you +started: + +```text +# MS Word documents +application/msword; ~/.mutt/view_attachment.sh %s "-" '/Applications/TextEdit.app' + +# Images +image/jpg; ~/.mutt/view_attachment.sh %s jpg +image/jpeg; ~/.mutt/view_attachment.sh %s jpg +image/pjpeg; ~/.mutt/view_attachment.sh %s jpg +image/png; ~/.mutt/view_attachment.sh %s png +image/gif; ~/.mutt/view_attachment.sh %s gif + +# PDFs +application/pdf; ~/.mutt/view_attachment.sh %s pdf + +# HTML +text/html; ~/.mutt/view_attachment.sh %s html + +# Unidentified files +application/octet-stream; ~/.mutt/view_attachment.sh %s "-" +``` + +The `view_attachment.sh` script is from [here][attachment]. Here's a link to +[my copy][attachment-mirror] in case that site ever goes down. Grab the script, +chmod it to executable, and stick it in `~/.mutt`. + +You can poke around and figure out how it works, or you can just not worry about +it and get on with life. I recommend the latter (at least for now). + +Now you can press `return` to open an attachment in the proper program. + +[attachment]: http://linsec.ca/Using_mutt_on_OS_X#mailcap +[attachment-mirror]: https://bitbucket.org/sjl/dotfiles/src/tip/mutt/view_attachment.sh + +### URLs + +One thing you'll probably want to do while reading email is open links. Many +terminal programs like iTerm2 let you command-click on a link to open it, but +this is Mutt! We shouldn't have to use the mouse! + +We're going to use a small helper program called urlview to make it easy to open +URLs in email. First, install it with `brew install urlview`. Then make +a `~/.urlview` file with the following contents: + +```text +COMMAND open %s +``` + +This tells urlview what command to use to open a URL. We're just going to use +the OS X `open` command to do the right thing. + +Next, add the following line to your `~/.muttrc`: + +```text +macro pager \Cu "|urlview" "call urlview to open links" +``` + +Now when you're reading an email with links in it you can press `Ctrl-u` to open +urlview. You'll see a screen like this: + +![urlview screen](/media/images/blog/2012/10/mutt-urls.png) + +Navigate with `j`, `k`, `gg`, `G`, or `/` and press `return` when the desired +link is selected. That link will be filled in at the bottom of the screen in +case you want to edit it, and you can press `return` one more time to actually +open it in your default browser. + +That about wraps it up for reading email. Now it's time to write some! + +Writing Email +------------- + +Writing email is one of the best parts of Mutt. First let's add a few settings +to get things nice and sane: + +```text +# Compose View Options ------------------------------- +set realname = "Steve Losh" # who am i? +set envelope_from # which from? +set sig_dashes # dashes before sig +set edit_headers # show headers when composing +set fast_reply # skip to compose when replying +set askcc # ask for CC: +set fcc_attach # save attachments with the body +unset mime_forward # forward attachments as part of body +set forward_format = "Fwd: %s" # format of subject when forwarding +set forward_decode # decode when forwarding +set attribution = "On %d, %n wrote:" # format of quoting header +set reply_to # reply to Reply to: field +set reverse_name # reply as whomever it was to +set include # include message in replies +set forward_quote # include message in forwards +``` + +You can reply to an email with `r` in the index or pager, or start a fresh one +with `m`. + +There's actually not a lot to say about writing mail, because Mutt itself +doesn't handle it! Mutt passes control off to the text editor of your choice. +Just specify your editor in your `~/.muttrc`: + +```text +set editor = "vim" # Use terminal Vim to compose email. +set editor = "mvim -f" # Use MacVim to compose email. +set editor = "subl -w" # Use Sublime Text 2 to compose email. +``` + +Any command that takes a filename and doesn't return until you're done can be +used here. + +This is fantastic because it means you can use an editor you're already +comfortable and fast in to write email instead of learning yet another set of +shortcuts. + +Once you save the email in your editor and close it, Mutt will present you with +a menu that looks like this: + +![Sending Screen](/media/images/blog/2012/10/mutt-send-1.png) + +You can press `e` to go back and edit the mail, `a` to add attachments, and so +on (the options are listed at the top of the screen). + +Before we can continue we need to tell Mutt how to send email. Press `q` to +discard the email for now. + +Sending Email +------------- + +Mutt does have (some) built-in SMTP support, but we're going to use a separate +program to do our sending for a few reasons. + +First, Mutt's SMTP support was considered "experimental" the last time +I checked. Sending email is kind of important, so we'll stick with something +tried and true. + +Second, we want a method that won't require our password in a plaintext config +file. + +Go ahead and install the `msmtp` program through Homebrew with `brew install +msmtp`. + +Next we're going to need to create a `~/.msmtprc` file with the following +contents: + +```text +account stevelosh +host smtp.gmail.com +port 587 +protocol smtp +auth on +from steve@stevelosh.com +user steve@stevelosh.com +tls on +tls_trust_file ~/.mutt/Equifax_Secure_CA.cert + +account default : stevelosh +``` + +`msmtp` will look in your keychain for your SMTP password, which we added +earlier. No plaintext passwords! + +The other "interesting" bit here is the `tls_trust_file`. We're going to be +connecting to Gmail's SMTP server over SSL, and `msmtp` needs to know if it can +trust the certificate that the server on the other end is sending back. + +Copy the following and paste it into the path `tls_trust_file` is set to: + +```text +-----BEGIN CERTIFICATE----- +MIIDIDCCAomgAwIBAgIENd70zzANBgkqhkiG9w0BAQUFADBOMQswCQYDVQQGEwJVUzEQMA4GA1UE +ChMHRXF1aWZheDEtMCsGA1UECxMkRXF1aWZheCBTZWN1cmUgQ2VydGlmaWNhdGUgQXV0aG9yaXR5 +MB4XDTk4MDgyMjE2NDE1MVoXDTE4MDgyMjE2NDE1MVowTjELMAkGA1UEBhMCVVMxEDAOBgNVBAoT +B0VxdWlmYXgxLTArBgNVBAsTJEVxdWlmYXggU2VjdXJlIENlcnRpZmljYXRlIEF1dGhvcml0eTCB +nzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAwV2xWGcIYu6gmi0fCG2RFGiYCh7+2gRvE4RiIcPR +fM6fBeC4AfBONOziipUEZKzxa1NfBbPLZ4C/QgKO/t0BCezhABRP/PvwDN1Dulsr4R+AcJkVV5MW +8Q+XarfCaCMczE1ZMKxRHjuvK9buY0V7xdlfUNLjUA86iOe/FP3gx7kCAwEAAaOCAQkwggEFMHAG +A1UdHwRpMGcwZaBjoGGkXzBdMQswCQYDVQQGEwJVUzEQMA4GA1UEChMHRXF1aWZheDEtMCsGA1UE +CxMkRXF1aWZheCBTZWN1cmUgQ2VydGlmaWNhdGUgQXV0aG9yaXR5MQ0wCwYDVQQDEwRDUkwxMBoG +A1UdEAQTMBGBDzIwMTgwODIyMTY0MTUxWjALBgNVHQ8EBAMCAQYwHwYDVR0jBBgwFoAUSOZo+SvS +spXXR9gjIBBPM5iQn9QwHQYDVR0OBBYEFEjmaPkr0rKV10fYIyAQTzOYkJ/UMAwGA1UdEwQFMAMB +Af8wGgYJKoZIhvZ9B0EABA0wCxsFVjMuMGMDAgbAMA0GCSqGSIb3DQEBBQUAA4GBAFjOKer89961 +zgK5F7WF0bnj4JXMJTENAKaSbn+2kmOeUJXRmm/kEd5jhW6Y7qj/WsjTVbJmcVfewCHrPSqnI0kB +BIZCe/zuf6IWUrVnZ9NA2zsmWLIodz2uFHdh1voqZiegDfqnc1zqcPGUIWVEX/r87yloqaKHee95 +70+sB3c4 +-----END CERTIFICATE----- +``` + +If you're paranoid and don't trust that I'm giving you the right cert (or that +someone has hacked my site and changed it), you can generate it yourself. I'll +leave that to you — if you care enough about it you'll figure it out. + +Now we need to tell Mutt to use msmtp. Add the following to your `~/.muttrc` +file: + +```text +set from = "steve@stevelosh.com" +set sendmail = "/usr/local/bin/msmtp -a stevelosh" +set sendmail_wait = 0 +unset record +``` + +The `-a stevelosh` will need to change to whatever you named your account in the +msmtp config. + +The `unset record` line tells Mutt to not append a copy of every email you send +to a file on your hard drive. Gmail will save the emails you send in the sent +folder, so you'll get the the next time you sync with offlineimap anyway. + +The `sendmail_wait` line tells Mutt to wait for the msmtp program to finish +sending the mail before returning control, instead of running it in the +background. This makes it obvious if there's a problem sending a message, which +I prefer to silent, backgrounded failures. + +Now you can send email! Awesome! + +Once you've composed a test email and saved it you'll be presented with a screen +like this (which we saw in the previous section): + +![Sending Screen](/media/images/blog/2012/10/mutt-send-1.png) + +The keys you need are listed along the top. Pressing `y` now will invoke msmtp +and send your email! + +You'll see "Sending message..." at the bottom of the screen while msmtp is +working. If there's a problem, Mutt will tell you the error. Figure it out +before moving on. + +Postponing Drafts +----------------- + +Sometimes I like to read and respond to email without an internet connection, +then actually send the replies when I get back to civilization. + +Since all of my email is stored locally, reading it offline is trivial. + +To reply or compose offline, I use Mutt's "postpone" feature. First I've added +the following line to my `~/.muttrc`: + +```text +bind compose p postpone-message +``` + +Now when I'm at the sending screen instead of pressing `y` to send I can press +`p` to postpone the message. This places the message in the drafts folder. + +The drafts folder in Mutt is just a normal folder of emails like any other. +When you sync with offlineimap your postponed email will get pushed up to Gmail +as a draft. You can edit it in the Gmail web interface if you like, and those +edits will sync back down too. + +If you want to edit your drafts (postponed messages) locally, you need to +"recall" them. I have the following line in my `~/.muttrc` to set the key to +the same one as I used to postpone it in the first place: + +```text +bind index p recall-message +``` + +You can also use `m` to start writing a new email, and mutt will prompt you if +there are existing postponed messages. + +Once you hit `p` (or `m` and then select "yes") Mutt will show you a list of the +postponed messages. Select one and press `return` to start editing it again. +If there's only one it will skip this step and simply open it for you +immediately. + +Once you're done editing you can either postpone it again or send it as usual. + +Unfortunately you have to go through the "recall → edit → send" process each +time. As far as I know there's no way to simply run down a list of postponed +emails sending each one with a single keystroke. But that's not *too* bad, and +it's great to be able to work offline like this. + +Once you send the postponed email (through Mutt or the Gmail web interface) it +disappears from the drafts folder and the postponed list as you would expect. + +Contacts +-------- + +Next we'll want to get Mutt to autocomplete our contacts from the OS X address +book. I like using the OS X address book because it automatically syncs between +my laptops and phone, so I only need to maintain one address list. + +### Autocompleting + +Unfortunately I've got some bad news for you: you're going to need to install +XCode. + +No, not the command-line developer tools. The full XCode. I'm sorry, but trust +me when I say it's going to save you a lot of pain, so just grumble to yourself +a bit and do it. + +Okay, now that you've got XCode you can install the `contacts` program through +Homebrew with `brew install contacts`. + +`contacts` is a command-line program that you can use to query your address +book. To tell Mutt how to use it add the following lines to your +`~/.mutt/muttrc`: + +```text +set query_command = "contacts -Sf '%eTOKEN%n' '%s' | sed -e 's/TOKEN/\t/g'" +bind editor complete-query +bind editor ^T complete +``` + +Now when you're filling out an email address field you can type a few characters +and hit Tab to get a screen like this: + +![Contacts](/media/images/blog/2012/10/mutt-contacts-1.png) + +You can use `j` and `k` to select an item, press return to complete it. Press +`q` if you've changed your mind and want to cancel the completion. Look at the +top of the screen for more handy little keys you can use here. + +If there's only one item in the list Mutt won't bother showing you this screen +and will just complete it right away. + +This completion searches more than just the email address. It'll also search +the names and possibly other fields from the address book entries as well. + +### Adding Contacts + +What about adding contacts to your address book? Any contacts you add on your +phone will automatically be synced, but what if you're reading your mail in Mutt +and just want to add the sender as a contact without leaving your command line? + +For this I use a little script that [Simone Manganelli][simx] wrote for me +called `addcontact`. You can [get it here][addcontact] and stick it in your +`$PATH` somewhere. It's just a command-line utility that you can use like this: + +```text +$ addcontact Steve Losh steve@stevelosh.com +$ addcontact "Steve Losh" steve@stevelosh.com +$ addcontact Steve Losh work steve@pculture.org +``` + +As you can see, it's pretty flexible. + +**Note:** This utility always adds a new contact record, so if you add someone +that's already in there you're going to get a duplicate entry. If that happens +you can search for the entries in Contacts.app, select the duplicates, and use +"Card → Merge Selected Cards" to combine them. + +Okay, so we can now add contacts from the command line. You could set up a Mutt +macro to automatically add senders without too much trouble, but I don't do +that. If I want to add a contact I just hit `!` and type out the shell command. +It's not that much work, and sometimes the name in the "From:" field is in +a weird format like "Last, First Middle" instead of "First Last", so this gives +me a chance to clean it up before I add it. + +[simx]: http://simx.me/ +[addcontact]: https://bitbucket.org/sjl/dotfiles/src/tip/bin/addcontact + +Searching Email +--------------- + +If you have more than a screen full of email in any given folder, you're going +to want a way to search through it. Mutt's built-in searching is a good start, +but I also use another program to get a bit more power. + +### Vanilla Searching + +There are two main ways to search your email in Mutt: plain searching and +"limiting". + +Plain searching is done with the `/` key. It works similarly to Vim or less: +you press `/`, type your query, and press enter to perform the search. You can +use `n` to move to the next match, and the search will loop around to the top if +it hits the bottom. + +I have the following lines in my `~/.muttrc` to bind `N` to go to the *previous* +match: + +```text +bind index N search-opposite +bind pager N search-opposite +``` + +Normally the `N` key marks a message as unread (or "new"). I personally never +want to do that. Unread mail should be "mail that has not been read". If you +use that feature you'll want to rebind it to something else. + +Two things about queries: + +1. They are regular expressions (actually it's more powerful than than, see [the + documentation][patterns] for more information). +2. They only search the To and Subject fields (*not* the message bodies!). + +I generally use this kind of searching when I see the email I want to open on +the screen but don't feel like pressing `j` or `k` forty times to move through +the list. When I'm actually trying to find a message I can't already see on the +screen I use limiting. + +[patterns]: http://www.mutt.org/doc/manual/manual-4.html#ss4.2 + +### Vanilla Limiting + +Limiting is the other way Mutt provides for searching mail. It's done with the +`l` key by default. + +Like `/`, `l` will ask you for a pattern. But instead of simply moving you to +the next message that matches the pattern, Mutt will *hide* messages that don't +match. + +This lets you see all the ones that match in a single list. This list works +just like a normal list. You can search it with `/`, tag things, and so on as +you normally would. It's really quite nice once you get used to it. + +To remove the limiting once you're done, you can limit to the special value +`all`. I've added a line to my `~/.muttrc` so I can do that with a single key: + +```text +macro index a "all\n" "show all messages (undo limit)" +``` + +**Note:** This shadows the `create-alias` function which I never use. + +Limit queries work exactly like search queries, so you can do powerful stuff +like `~f arthur ~C honza ~s api` ("limit to messages from 'arthur', to or cced +to 'honza', containing 'api' in the subject"). + +### Full-Text Searching + +By now you're probably wondering how to search the full text of messages. There +are two ways: one simple and slow, the other complicated and fast. + +First is the simple, slow way. You can use `~B` in your searches and limits to +search inside the entire message. If your folder only has a hundred messages +this works great. But once you have a few more (my archive has about 30,000 at +the moment, and I prune it fairly often) it quickly becomes too slow to be +usable. + +I use a program called [notmuch][] to index and search my email. It's blazingly +fast and works pretty well. + +First, install it with `brew install notmuch`. Now you need +a `~/.notmuch-config` file. Run `notmuch setup` to generate one. It's pretty +straightforward. When it asks you for the path to your archive, that's the path +to the folder containing all your individual IMAP folders +(`~/.mail/steve-stevelosh.com` in my setup). + +Now run `notmuch new` to perform the initial index. It might take a while if +you have a lot of folders. Subsequent indexes will be much faster since they'll +only reindex whatever changed since the last time. + +You'll want to reindex when your email changes, so add a hook in your +`~/.offlineimaprc` to run `notmuch new` whenever offlineimap syncs your email: + +```text +[Account SteveLosh] +localrepository = SteveLosh-Local +remoterepository = SteveLosh-Remote +status_backend = sqlite +postsynchook = notmuch new +``` + +Now you can use `notmuch search foo` to search your mail for "foo" from the +command line. Try a couple of queries and make sure it works. + +Now we need to hook notmuch into Mutt. There are a number of different ways to +do this, all of them hacky and ugly. I'll describe how I do it. + +A quick overview of how this is going to work: + +1. You'll press a key in Mutt to activate searching. +2. You'll type your query and press return. +3. Your mail will be searched with `notmuch`. +4. Τhe resulting messages will be symlinked into a temporary maildir folder. +5. That temporary folder will be opened in Mutt. + +First get [mutt-notmuch-py][]. The original `mutt-notmuch` is a Perl script +with many external requirements that are a pain to install, and it doesn't work +on OS X. `mutt-notmuch-py` is a Python script with zero external requirements. + +`mutt-notmuch-py` is going to handle steps 2 through 4 in the list. Get the +script into your `$PATH` somehow and then run it: + +```text +$ mutt-notmuch-py -G ~/.mail/temporary/search +Query: foo + +$ ls ~/.mail/temporary/search +cur new +``` + +The `-G` tells it to not perform certain Gmail-specific stuff that we don't +need. The path is where it will create the temporary maildir folder with the +results. Each time you run it this folder will be wiped clean before the new +results are linked into it. + +Now we need to handle the first and last points in the list. I have the +following mapping in my `~/.muttrc`: + +```text +macro index S "unset wait_keymutt-notmuch-py ~/.mail/temporary/search+temporary/search" "search mail (using notmuch)" +``` + +That's a lot to take in, so let's see how it works piece by piece: + +```text +macro index S +``` + +We're going to use the `S` key to perform a full search of all of our mail. + +```text +unset wait_key +``` + +Unset the `wait_key` Mutt option to prevent Mutt from asking us to press a key +after the search has finished. + +```text +mutt-notmuch-py -G ~/.mail/temporary/search +``` + +Run `mutt-notmuch-py`. Control will pass over to that script and it will ask +you for your query, run the search, set up the results folder, and then hand +control back to Mutt. + +```text ++temporary/search +``` + +Now we change over to the temporary folder in Mutt, and we're now looking at +a list of search results! Since this is a real Maildir folder like any other +one we can use the full range of tools to interact with it (searching, limiting, +etc). + +Finally, let's get this search results folder in our sidebar so we can see where +we are visually at all times: + +```text +mailboxes +steve-stevelosh.com/INBOX \ + +steve-stevelosh.com/vim \ + +steve-stevelosh.com/clojure \ + ... + +temporary/search \ +``` + +Now the search results folder can be navigated like any other one. That's it +for email searching! Now you should have a setup that you can use in real life +to manage your email. + +[notmuch]: http://notmuchmail.org/ +[mutt-notmuch-py]: https://github.com/honza/mutt-notmuch-py + +Conclusion +---------- + +Mutt is definitely a beast. It's old, crufty, and ugly, but if you spend the +time to set it up and learn to use it you'll be rewarded with a fast, powerful, +customizable environment for working with your email. + +A wonderful trend these days is that more and more sites are including the +ability to respond to comments and such by simply replying to their notification +emails. This means that often you can reply to Facebook emails, comment on +GitHub pull requests, and respond to Bitbucket issues all without leaving the +comfort of your finely-tuned email client. + +Mutt's not for everyone, but if you deal with a lot of email and have some time +to spend you should give it a try. You just might learn to love the old dog. + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/10/why-i-two-space.html --- a/content/blog/2012/10/why-i-two-space.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,214 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "Why I Two-Space" - snip: "You can pry my extra spaces from my cold, dead hands." - created: 2012-10-12 10:10:00 - %} - -{% block article %} - -If you look at [the source code][source] for this blog, you might notice that -all my blog posts (written in [Markdown][]) have two spaces after every period. - -Every so often [this Slate article][slate] makes the rounds and annoys me. This -time I figured I'd write a blog post/rant to get it off my chest once and for -all: two-spacing is equal or superior to one-spacing in all non-trivial ways. - -[source]: https://bitbucket.org/sjl/stevelosh/src/tip/content/blog -[Markdown]: http://daringfireball.net/projects/markdown/ -[slate]: http://www.slate.com/articles/technology/technology/2011/01/space_invaders.html - -[TOC] - -The Arguments for One-Spacing ------------------------------ - -If you read the aforementioned Slate article you'll see that it presents a total -of three arguments in favor of single-spacing after a period: - -1. Many professional typographers find two-spacing ugly (though they have no - actual evidence of its effects on readability). -2. It's less work to press the space bar once instead of twice. -3. It's pretty much arbitrary anyway, so *why not*? - -You might also infer from the article's tone that "you get to feel superior to -those Neanderthals that use two spaces" is another argument in favor of it, but -I'll leave that one out because it can go both ways. - -Let's go through these arguments one by one and see how they hold up. - -Effort ------- - -I don't think the "it's more effort to press the space bar twice" argument is at -all compelling. The extra space *may* account for around half a percent of the -keystrokes you type, depending on how long your sentences tend to run (mine -usually end up fairly long, as you can see). - -Not only is it not a large portion of your keystrokes, but the space bar has -your two strongest digits dedicated solely to it! If any fingers are going to -wear out from typing, it won't be your thumbs. - -You could get pedantic and mention increased file size as a disadvantage of two -spacing, but let's not quibble about a few bytes when I can hold 32 gigabytes on -an SD card the size of a postage stamp. - -Two-Spacing is Ugly -------------------- - -The Slate article is correct is saying that many professional typographers think -two-spacing is ugly. My copy of [The Elements of Typographic Style][elements] -agrees. However, there's more to the issue than a single blanket rule. - -[elements]: http://www.amazon.com/dp/0881792063/?tag=stelos-20 - -### Source and Presentation - -If you're reading this in a web browser and look closely, you'll notice that -there's only one space between each sentence. I said I used two, so what -happened? - -Web browsers, by default, collapse successive whitespace into a single space when it -renders the HTML. You could put a thousand spaces between sentences in the -source and it would still come out single-spaced. - -Another widely-used format, LaTeX, splits the difference and actually uses -somewhere between one and two spaces (by default). - -The key idea here is that what you type and what the end user actually reads are -two different things. They don't need to be bound together. It's possible to -type two spaces but get one in your output (or in LaTeX's case: an even more -pleasing "one and a bit"). - -If you're using a WYSIWYG editor like Microsoft Word this may not be the case. -For the high school students typing up papers for class: sure, go ahead and -single-space. But if you're still using Word to type up things like books, long -blog posts, or technical documentation, you're doing yourself a disservice. -Learn to use a system like LaTeX so your source and rendered output aren't -locked together. - -This means that "two-spacing looks ugly" doesn't imply "you should not type two -spaces when you write". It *actually* implies "you should use a system that -results in single-spacing when rendered", and most of the common ones today will -do exactly that. - -### Typewriters - -So we've seen that the "extra spaces take more effort" argument isn't -convincing, and that "two-spacing in the final output is ugly" doesn't prevent -you from using two spaces in the input. - -But so far I haven't given you any reason *for* using two spaces. That's the -Slate article's third argument: "it's arbitrary anyway, so you may as well use -one". - -Let's fix that by examining the common holier-than-thou put-down of "two-spacing -was useful back when people used *typewriters*, you dinosaur!". - -To his credit, the Slate author correctly points out that it's not the fact that -people used typewriters that made two-spacing popular, it was the fact that -*typewriters used monospaced fonts*. Most people miss that logical leap. - -Unfortunately he follows that up with: - -> Here's the thing, though: Monospaced fonts went out in the 1970s. - -For the average person typing up a school paper here and there, sure. But for -many people doing a large amount of writing for LaTeX books and papers, -technical documentation, code comments, mailing lists, blog posts, and lots of -other things, it's simply *wrong* for one big reason: - -**Text editors use monospaced fonts!** - -It doesn't matter what editor you use. Vim, Emacs, TextMate, Sublime Text, -Eclipse, Gedit, Notepad++? *All* of them use monospaced fonts. - -Going back to the typewriter quip, this means that if people used two spaces -when writing on typewriters because it looked better, then using two spaces in -a text editor will look better. And we know that because there's a distinction -between source and presentation in non-WYSIWYG contexts, we can use separate -strategies for each. - -We can have our cake and eat it too! We can type two spaces after a period to -make our text look better as we write, revise, and edit it, and then render it -to single-spacing (or "space-and-a-half-ing") to give our readers a beautiful -reading experience with pleasant spacing. - -So now two-spacing has a real advantage over single-spacing. That's enough to -make it preferable. But let's look at one more advantage for power users, just -to seal the deal. - -Power ------ - -If you use [Vim][] to edit text, you're probably familiar with its "text -objects". Text objects are what let you move and act on whole chunks of text at -a time. For example, instead of deleting a word letter-by-letter you can use -`daw` to "delete around word". - -Vim comes with a "sentence" text object built-in. You can move around your -document sentence-by-sentence with `(` and `)`, and yank/delete/change/etc -entire sentences with `cas` ("change around sentence") and so on. - -You can probably guess where this is going. If you single-space sentences Vim -will do its best to "do the right thing", but inevitably gets tripped up when -you've got punctuation in your sentence. For example: - - Bob started speaking. Hello, Mr. Smith! How are you today? - ^ - cursor - -What happens when you tell Vim to "delete around sentence" now? - - Bob started speaking. Hello, Mr. How are you today? - ^ - cursor - -Well that's not right! Vim can't easily tell the difference between the period -after "Mr" and the end of a sentence. What happens if you type your prose with -two-spacing instead? - - Bob started speaking. Hello, Mr. Smith! How are you today? - ^ - cursor - Bob started speaking. How are you today? - ^ - cursor - -This time Vim is able to delete the sentence correctly! Note that you'll need -to make sure to `set cpo+=J` in your `~/.vimrc` file to tell Vim "don't worry, -I'm using two spaces like a sane person" for this to work. - -Two-spacing provides more semantic information, which means that software can -parse and work with it more easily. I'm sure Emacs has something similar, and -if you or someone else ever needs to parse your writing programatically they'll -have an easier time. - -Final Score ------------ - -To recap, the arguments *for* single-spacing are: - -1. Two-spacing is ugly in proportional fonts. -2. It's less work to press the space bar once instead of twice. -3. It's pretty much arbitrary anyway, so why bother with two? - -Number 1 is irrelevant, because writing and rendering are (except in trivial -cases) two orthogonal activities. - -Number 2 isn't very convincing. - -Number 3 is false, because two-spacing gives you two advantages over -one-spacing: - -1. It looks better in your editor. -2. It gives you more power when editing and parsing. - -So the next time you see that arrogant Slate article, feel free to be arrogant -right back. Wield your extra spaces proudly, because they give you both comfort -and power! - -[Vim]: http://www.vim.org/ - -{% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2012/10/why-i-two-space.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2012/10/why-i-two-space.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,211 @@ ++++ +title = "Why I Two-Space" +snip = "You can pry my extra spaces from my cold, dead hands." +date = 2012-10-12T10:10:00Z +draft = false + ++++ + +If you look at [the source code][source] for this blog, you might notice that +all my blog posts (written in [Markdown][]) have two spaces after every period. + +Every so often [this Slate article][slate] makes the rounds and annoys me. This +time I figured I'd write a blog post/rant to get it off my chest once and for +all: two-spacing is equal or superior to one-spacing in all non-trivial ways. + +[source]: https://bitbucket.org/sjl/stevelosh/src/tip/content/blog +[Markdown]: http://daringfireball.net/projects/markdown/ +[slate]: http://www.slate.com/articles/technology/technology/2011/01/space_invaders.html + +{{% toc %}} + +The Arguments for One-Spacing +----------------------------- + +If you read the aforementioned Slate article you'll see that it presents a total +of three arguments in favor of single-spacing after a period: + +1. Many professional typographers find two-spacing ugly (though they have no + actual evidence of its effects on readability). +2. It's less work to press the space bar once instead of twice. +3. It's pretty much arbitrary anyway, so *why not*? + +You might also infer from the article's tone that "you get to feel superior to +those Neanderthals that use two spaces" is another argument in favor of it, but +I'll leave that one out because it can go both ways. + +Let's go through these arguments one by one and see how they hold up. + +Effort +------ + +I don't think the "it's more effort to press the space bar twice" argument is at +all compelling. The extra space *may* account for around half a percent of the +keystrokes you type, depending on how long your sentences tend to run (mine +usually end up fairly long, as you can see). + +Not only is it not a large portion of your keystrokes, but the space bar has +your two strongest digits dedicated solely to it! If any fingers are going to +wear out from typing, it won't be your thumbs. + +You could get pedantic and mention increased file size as a disadvantage of two +spacing, but let's not quibble about a few bytes when I can hold 32 gigabytes on +an SD card the size of a postage stamp. + +Two-Spacing is Ugly +------------------- + +The Slate article is correct is saying that many professional typographers think +two-spacing is ugly. My copy of [The Elements of Typographic Style][elements] +agrees. However, there's more to the issue than a single blanket rule. + +[elements]: http://www.amazon.com/dp/0881792063/?tag=stelos-20 + +### Source and Presentation + +If you're reading this in a web browser and look closely, you'll notice that +there's only one space between each sentence. I said I used two, so what +happened? + +Web browsers, by default, collapse successive whitespace into a single space when it +renders the HTML. You could put a thousand spaces between sentences in the +source and it would still come out single-spaced. + +Another widely-used format, LaTeX, splits the difference and actually uses +somewhere between one and two spaces (by default). + +The key idea here is that what you type and what the end user actually reads are +two different things. They don't need to be bound together. It's possible to +type two spaces but get one in your output (or in LaTeX's case: an even more +pleasing "one and a bit"). + +If you're using a WYSIWYG editor like Microsoft Word this may not be the case. +For the high school students typing up papers for class: sure, go ahead and +single-space. But if you're still using Word to type up things like books, long +blog posts, or technical documentation, you're doing yourself a disservice. +Learn to use a system like LaTeX so your source and rendered output aren't +locked together. + +This means that "two-spacing looks ugly" doesn't imply "you should not type two +spaces when you write". It *actually* implies "you should use a system that +results in single-spacing when rendered", and most of the common ones today will +do exactly that. + +### Typewriters + +So we've seen that the "extra spaces take more effort" argument isn't +convincing, and that "two-spacing in the final output is ugly" doesn't prevent +you from using two spaces in the input. + +But so far I haven't given you any reason *for* using two spaces. That's the +Slate article's third argument: "it's arbitrary anyway, so you may as well use +one". + +Let's fix that by examining the common holier-than-thou put-down of "two-spacing +was useful back when people used *typewriters*, you dinosaur!". + +To his credit, the Slate author correctly points out that it's not the fact that +people used typewriters that made two-spacing popular, it was the fact that +*typewriters used monospaced fonts*. Most people miss that logical leap. + +Unfortunately he follows that up with: + +> Here's the thing, though: Monospaced fonts went out in the 1970s. + +For the average person typing up a school paper here and there, sure. But for +many people doing a large amount of writing for LaTeX books and papers, +technical documentation, code comments, mailing lists, blog posts, and lots of +other things, it's simply *wrong* for one big reason: + +**Text editors use monospaced fonts!** + +It doesn't matter what editor you use. Vim, Emacs, TextMate, Sublime Text, +Eclipse, Gedit, Notepad++? *All* of them use monospaced fonts. + +Going back to the typewriter quip, this means that if people used two spaces +when writing on typewriters because it looked better, then using two spaces in +a text editor will look better. And we know that because there's a distinction +between source and presentation in non-WYSIWYG contexts, we can use separate +strategies for each. + +We can have our cake and eat it too! We can type two spaces after a period to +make our text look better as we write, revise, and edit it, and then render it +to single-spacing (or "space-and-a-half-ing") to give our readers a beautiful +reading experience with pleasant spacing. + +So now two-spacing has a real advantage over single-spacing. That's enough to +make it preferable. But let's look at one more advantage for power users, just +to seal the deal. + +Power +----- + +If you use [Vim][] to edit text, you're probably familiar with its "text +objects". Text objects are what let you move and act on whole chunks of text at +a time. For example, instead of deleting a word letter-by-letter you can use +`daw` to "delete around word". + +Vim comes with a "sentence" text object built-in. You can move around your +document sentence-by-sentence with `(` and `)`, and yank/delete/change/etc +entire sentences with `cas` ("change around sentence") and so on. + +You can probably guess where this is going. If you single-space sentences Vim +will do its best to "do the right thing", but inevitably gets tripped up when +you've got punctuation in your sentence. For example: + + Bob started speaking. Hello, Mr. Smith! How are you today? + ^ + cursor + +What happens when you tell Vim to "delete around sentence" now? + + Bob started speaking. Hello, Mr. How are you today? + ^ + cursor + +Well that's not right! Vim can't easily tell the difference between the period +after "Mr" and the end of a sentence. What happens if you type your prose with +two-spacing instead? + + Bob started speaking. Hello, Mr. Smith! How are you today? + ^ + cursor + Bob started speaking. How are you today? + ^ + cursor + +This time Vim is able to delete the sentence correctly! Note that you'll need +to make sure to `set cpo+=J` in your `~/.vimrc` file to tell Vim "don't worry, +I'm using two spaces like a sane person" for this to work. + +Two-spacing provides more semantic information, which means that software can +parse and work with it more easily. I'm sure Emacs has something similar, and +if you or someone else ever needs to parse your writing programatically they'll +have an easier time. + +Final Score +----------- + +To recap, the arguments *for* single-spacing are: + +1. Two-spacing is ugly in proportional fonts. +2. It's less work to press the space bar once instead of twice. +3. It's pretty much arbitrary anyway, so why bother with two? + +Number 1 is irrelevant, because writing and rendering are (except in trivial +cases) two orthogonal activities. + +Number 2 isn't very convincing. + +Number 3 is false, because two-spacing gives you two advantages over +one-spacing: + +1. It looks better in your editor. +2. It gives you more power when editing and parsing. + +So the next time you see that arrogant Slate article, feel free to be arrogant +right back. Wield your extra spaces proudly, because they give you both comfort +and power! + +[Vim]: http://www.vim.org/ + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2013/03/list-out-of-lambda.html --- a/content/blog/2013/03/list-out-of-lambda.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1065 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "List Out of Lambda" - snip: "Down the rabbit hole we go!" - created: 2013-03-30 14:00:00 - %} - - {% block article %} - -If you ignore the practical issues of computers like size, weight, cost, heat, -and so on, what do you *really* need in a programming language? Let's play -around with this question. - -To understand this post you'll need a basic understanding of how functions in -Javascript work. If you can look at this code and understand what it prints, -you're good to go: - - :::javascript - var x = 10; - - var f = function(y) { - console.log(x); - console.log(y); - } - - var g = f; - - f(1); - g(2); - -This blog post is a thought exercise. It's not something you'd ever use for -real code. But just like a guitarist practices scales that she won't ever play -in a song, we programmers should be exercising our brains every so often. - -I'm going to use Javascript for the examples. Any language with first class -functions and lexical scoping (basically: closures) will work. The examples -would be prettier in a Lisp, but some people would be turned off by the syntax -and miss out on some interesting ideas. Feel free to port the code if it -bothers you. - -If you've already seen this kind of thing before (maybe you've gone through [The -Little Schemer][] or [SICP][]) you may want to just skim the code here and look -for anything new. - -If you *haven't* seen anything like this, then you're in for a treat! It's all -going to look extremely weird the first time you see it. Go slowly and make -sure you understand each piece fully before moving on to the next. These -concepts may be unintuitive, but they're built from very simple pieces. - -Finally: if you get stuck, don't worry. Tracing out the execution of a function -on paper can be a good way to wrap your brain around it (I recommend investing -in a good lap desk for comfy reading and writing). If that doesn't work, just -close the window and come back tomorrow. Sometimes new concepts need a while to -rattle around in your brain before they click into place. - -[The Little Schemer]: http://www.amazon.com/dp/0262560992/?tag=stelos-20 -[SICP]: http://www.amazon.com/dp/0070004846/?tag=stelos-20 - -[TOC] - -Lists ------ - -Let's get started. One of the most common things we do as programmers is -grouping data together. Javascript has "arrays" built in to the language for -this: - - :::javascript - var names = ["Alice", "Bob", "Candice"]; - -What if Javascript didn't come with arrays included? Could we create them (or -something like them) ourselves? - -To answer this, let's think about the bare minimum we'd need to "bootstrap" -something like an array. There are a number of ways to do this, but we're going -to look at one in particular. - -We'll call our array-like thing a "list". To make it work, we need four parts: - -* The concept of "the empty list". -* A way to add an element to the front of a list. -* A way to take a list and get the first element. -* A way to take a list and get everything *but* the first element. - -If we have those four things, we can build on top of them to do anything else we -might want. For example: to make a list of one item, you add that item to the -front of the empty list. - -Let's narrow this down further. There are lots of ways you could implement -those four things -- I'm going to use functions. Let's sketch out an outline: - - :::javascript - var empty_list = null; - - var prepend = function(el, list) { - // ... - }; - - var head = function(list) { - // ... - }; - - var tail = function(list) { - // ... - }; - - var is_empty = function(list) { - // ... - }; - -Here are the descriptions of each of these items. - -The `empty_list` is a special value that represents a list of zero elements. -It can be anything, so for now we'll use `null` (we'll revisit this later). - -`prepend(1, some_list)` will return a new list that looks like the old one, but -with `1` stuck onto the front of it. So if we want to create a list of the -numbers `1` and `2` we can say `prepend(1, prepend(2, empty_list))` or "prepend -one to the result of prepending 2 to the empty list". - -`head(some_list)` will return the first element in the list. Calling it on the -empty list will be undefined, so we'll just be very careful not to do that! - -`tail(some_list)` will return a new list that's like the one we gave it, but -with the first element removed. Again, calling this on an empty list will make -things explode. - -`is_empty(some_list)` will return `true` if the list given to it is the empty -list, and `false` otherwise. - -Once we have those four functions (plus the special empty list value) we can -start building on top of them, so let's figure out how to make them! - -### List Out of If - -If you haven't seen anything like this before, you might think it's time to -start creating Javascript Objects. That's certainly one way to do it. - -Since this post is a thought experiment in what we actually *need*, though, -let's try to avoid using big language features (like Objects) unless we -absolutely can't avoid it. - -So if we don't want to use other language features yet, what are we left with? -Well so far our skeleton only has functions (and `null`), so let's try those! - -Here's the first working revision of the building blocks of lists: - - :::javascript - var empty_list = null; - - var prepend = function(el, list) { - return function(command) { - if (command === "head") { - return el; - } else if (command === "tail") { - return list; - } - } - }; - - var head = function(list) { - return list("head"); - }; - - var tail = function(list) { - return list("tail"); - }; - - var is_empty = function(list) { - return list === null; - }; - -Go ahead and paste that into a browser console and play with it: - - :::javascript - var e = empty_list; - - console.log(is_empty(e)); - // true - - var names = prepend("Alice", - prepend("Bob", - prepend("Candice", - empty_list))); - - console.log(is_empty(names)); - // False - - console.log(head(names)); - // Alice - - console.log(tail(names)); - // Some function representing the list of ("Bob", "Candice") - - console.log(head(tail(names))); - // Bob - -### But Where is the Data? - -Did the definitions of those functions surprise you? Lists seem like such an -important, object-oriented concept, but there only appear to be functions here! - -Let's look at how this actually works. First of all, the "empty list" concept -is pretty straightforward: - - :::javascript - var empty_list = null; - - var is_empty = function(list) { - return list === null; - }; - -We could have picked any arbitrary value here. `null` seemed appropriate, so -I used that. - -Now on to the meat of things: `prepend`. - - :::javascript - var prepend = function(el, list) { - return function(command) { - if (command === "head") { - return el; - } else if (command === "tail") { - return list; - } - } - }; - -This is where the real magic happens. Let's think through it. - -First of all, we know that when you prepend something to a list, you're going to -get a (new) list back. So whatever `prepend` returns must be a list. - -Looking at the code, we can see it returns a function. So in our little thought -experiment, a list is actually a Javascript function under the hood! - -So what do we need to do with lists (aside from empty checking, which we've -already covered)? Well, we need to be able to get the head and the tail. When -we call `prepend(h, t)`, we happen to be conveniently specifying the head and -tail as the arguments! So in `prepend` we return a function that knows how to -return its own head or tail when asked. - -So a "list" is "a function that knows how to return its own head or tail when -asked". So our `head` and `tail` functions just need to ask nicely! - - :::javascript - var head = function(list) { - return list("head"); - }; - - var tail = function(list) { - return list("tail"); - }; - -That's it! We've created lists in 23 lines of code without using any fancy -things like Objects. Before you move on, make sure you really understand why -this works. Write out a few examples on paper. - - :::javascript - var empty_list = null; - - var prepend = function(el, list) { - return function(command) { - if (command === "head") { - return el; - } else if (command === "tail") { - return list; - } - } - }; - - var head = function(list) { - return list("head"); - }; - - var tail = function(list) { - return list("tail"); - }; - - var is_empty = function(list) { - return list === null; - }; - -### Building on the Foundations - -Now that we have lists, let's implement a few common things on top of them as -practice. - -#### map - -A common thing to do to a list is to create a new list by looping through it and -doing something to each item. This is called "map": - - :::javascript - var map = function(fn, l) { - if (is_empty(l)) { - return empty_list; - } else { - return prepend(fn(head(l)), map(fn, tail(l))); - } - }; - -If you're not used to recursive definitions like this, you may way to take -a few minutes and try to work out how it works. Here's an example: - - :::javascript - var square = function(x) { - return x * x; - } - var numbers = prepend(1, prepend(2, prepend(3, empty_list))); - - var squared_numbers = map(square, numbers); - // map(square, [1, 2, 3]) - // prepend(square(1), map(square, [1, 2, 3])) - // prepend(square(1), prepend(square(2), map(square, [3]))) - // prepend(square(1), prepend(square(2), prepend(square(3), map(square, [])))) - // prepend(square(1), prepend(square(2), prepend(square(3), []))) - // prepend(square(1), prepend(square(2), prepend(9, []))) - // prepend(square(1), prepend(square(2), [9])) - // prepend(square(1), prepend(4, [9])) - // prepend(square(1), [4, 9]) - // prepend(1, [4, 9]) - // [1, 4, 9] - -I'm using brackets here to represent lists, but remember that these aren't -arrays, but are actually the functions that were returned by `prepend`. - -If you're still not sure about this, trace out every step of `map(square, -empty_list)` on paper. Then trace out every step of `map(square, prepend(10, -empty_list))`. - -Thinking recursively like this takes some practice. I have notebooks filled -with [pages like this](http://i.imgur.com/kqu5jy9.jpg). Experienced guitarists -practice new material slowly and methodically -- there's no reason programmers -shouldn't do the same. Watching the function calls expand and contract on paper -can help you feel in your gut how these things work in a way that just staring -at the words can't. - -#### filter - -We're going to start moving a bit faster now, but you should still make sure you -understand everything completely before moving on. Take as much time as you -need. Write things out. Run them. Get a feel for them. - -The next "utility" function we'll build on top of lists is `filter`, which -takes a function and a list, and returns a new list whose elements are those in -the original that make the function return `true`. Here's an example: - - :::javascript - var numbers = prepend(1, prepend(2, prepend(3, empty_list))); - var is_odd = function(x) { - return x % 2 === 1; - } - - filter(is_odd, numbers); - // [1, 3] - -Now let's implement `filter`: - - :::javascript - var filter = function(fn, l) { - if (is_empty(l)) { - return empty_list; - } else if (fn(head(l))) { - return prepend(head(l), filter(fn, tail(l))); - } else { - return filter(fn, tail(l)); - } - }; - -Take your time. Trace out some examples. Move on when you feel it in your gut. - -#### and, or, not - -Let's take a slight detour to implement a few "helper" functions. Τhese don't -have anything specifically to do with lists, but we'll need them later. - - :::javascript - var not = function(x) { - if (x) { - return false; - } else { - return true; - } - }; - - var and = function(a, b) { - if (a) { - if (b) { - return true; - } else { - return false; - } - } else { - return false; - } - }; - - var or = function(a, b) { - if (a) { - return true; - } else { - if (b) { - return true; - } else { - return false; - } - } - }; - -Javascript already has these things built in as `!`, `&&`, and `||`, of course, -but remember that in this thought exercise we're trying to avoid using extra -language features if we don't need them. How far can we scrape by on just -functions and `if` statements? - -One small note: these functions are just normal Javascript functions, which -means that `and(a, b)` will *not* short-circuit like `a && b` would. For our -purposes here that won't hurt us, but it's something to be aware of. - -### List Out of Lambda - -Now that we've had a bit more practice, let's go back to our definition of -lists: - - :::javascript - var empty_list = null; - - var prepend = function(el, list) { - return function(command) { - if (command === "head") { - return el; - } else if (command === "tail") { - return list; - } - } - }; - - var head = function(list) { - return list("head"); - }; - - var tail = function(list) { - return list("tail"); - }; - - var is_empty = function(list) { - return list === null; - }; - -There are a few things about this implementation that bother me. Our goal is to -use as few language features as possible, but we've actually used quite a few! -I count at least five: - -* Functions -* `if` statements -* Strings -* Booleans (the `true`/`false` result of `is_empty`) -* Equality checking (the `===` checks) - -It turns out we can remove most of those things at the cost of a bit of -readability (and more bending of our minds). - -Let's start by rewriting the core three functions to ditch those ugly strings, -equality checks, and even the `if` statement: - - :::javascript - var prepend = function(el, list) { - return function(selector) { - return selector(el, list); - }; - }; - - var head = function(list) { - return list(function(h, t) { - return h; - }); - }; - - var tail = function(list) { - return list(function(h, t) { - return t; - }); - }; - -You may want to get a snack before wrapping your brain around this one! There's -no strings, no equality checking, no `if` statements. But we still have lists! - -The `prepend` functions still returns a function, just like before. Remember -that in the last implementation, a "list" was really a function that knew how to -give out its head or its tail when asked for them. - -This time, we're inverting the "asking". In this version, a "list" is "a -function that will tell another function about both its head *and* its tail when -asked". This time the *asker* gets *both* pieces, and can decide which one they -want to use. - -Let's look at the `head` function: - -* `head` takes a list and says `return list(...)`, which means: "Hey list, - I would like you to tell all of your info to this little helper function I'm - giving you". -* The list says `return ...(el, list)`, which means: "Okay helper function, - here's my head and my tail, enjoy!" -* The helper function that `head` originally gave was `function(h, t) { return - h; }`. So when the list calls it with the head and the tail as arguments, it - returns the head and ignores the tail. -* `head` takes that result and just returns it straight through back to the - caller. - -`tail` works exactly the same way, but its helper function returns the second -argument (the tail) instead of the first. - -That's it! The equality checking and `if` statements have disappeared. Can you -describe where they've gone? What has taken their place? - -Before we move on, let's clean up the idea of the empty list. It's still using -`null` and equality checking. Let's remove those and make things a little more -uniform. - -To do this we'll need to change the other three functions a bit as well, but if -you've understood everything so far it shouldn't be too bad. - - :::javascript - var empty_list = function(selector) { - return selector(undefined, undefined, true); - }; - - var prepend = function(el, list) { - return function(selector) { - return selector(el, list, false); - }; - }; - - var head = function(list) { - return list(function(h, t, e) { - return h; - }); - }; - - var tail = function(list) { - return list(function(h, t, e) { - return t; - }); - }; - - var is_empty = function(list) { - return list(function(h, t, e) { - return e; - }); - }; - -We've now made lists a bit smarter. In addition to telling the helper function -their head and tail, they also tell it "am I the empty list?". We've modified -the helpers `head` and `tail` to accept (and ignore) this extra argument. - -We then modified `is_empty` to work just like `head` and `tail`. - -Finally, we've redefined `empty_list` to match the rest of the lists instead of -being a special, magic value. The empty list is now just like a normal one: -it's a function that takes an "asker" and tells that asker "Hey, my head and -tail are undefined and I am the empty list". - -I used `undefined` here which is technically another language feature because -it's easier to read. Feel free to replace it with anything you want to make it -more pure. Since we're being very careful to never call `head` or `tail` on the -empty list those values will never be seen anyway. - -So after all that, we've finally implemented the building blocks of lists with -only two things: - -* Functions. -* `true` and `false` for empty lists. - -If you're up for a challenge, think about whether you could remove that second -item (and if so, are you *really* removing it, or just using certain features of -Javascript implicitly instead of explicitly?). - -A Brief Intermission --------------------- - -Let's take a minute to reflect on all the code we've seen so far. First, we -have an implementation of lists that uses only functions and booleans: - - :::javascript - var empty_list = function(selector) { - return selector(undefined, undefined, true); - }; - - var prepend = function(el, list) { - return function(selector) { - return selector(el, list, false); - }; - }; - - var head = function(list) { - return list(function(h, t, e) { - return h; - }); - }; - - var tail = function(list) { - return list(function(h, t, e) { - return t; - }); - }; - - var is_empty = function(list) { - return list(function(h, t, e) { - return e; - }); - }; - -From this point on, we can now ignore the details of how lists are implemented. -As long as we have `head`, `tail`, and `prepend` we don't need to worry about -what lists actually *are* under the hood. - -We also built a few helper functions on top of this foundation: - - :::javascript - var not = function(x) { - if (x) { - return false; - } else { - return true; - } - }; - - var and = function(a, b) { - if (a) { - if (b) { - return true; - } else { - return false; - } - } else { - return false; - } - }; - - var or = function(a, b) { - if (a) { - return true; - } else { - if (b) { - return true; - } else { - return false; - } - } - }; - - var map = function(fn, l) { - if (is_empty(l)) { - return empty_list; - } else { - return prepend(fn(head(l)), map(fn, tail(l))); - } - }; - - var filter = function(fn, l) { - if (is_empty(l)) { - return empty_list; - } else if (fn(head(l))) { - return prepend(head(l), filter(fn, tail(l))); - } else { - return filter(fn, tail(l)); - } - }; - -Before you move on, make sure all of this code is crystal clear. Come back -tomorrow if you need to let it sink in. We're about to go a lot deeper into the -rabbit hole, so make sure you're ready. - -Numbers -------- - -If you look at the definitions of `prepend`, `head`, and `tail`, they're pretty -mind-bending. However, the definitions of `map` and `filter` are much more -straightforward. - -This is because we encapsulated the implementation of lists into the first four -functions. We did all the hard work of building lists out of almost nothing at -all and hid it behind that simple `prepend`, `head`, and `tail` interface. - -The idea of creating things from simple pieces and abstracting them into "black -boxes" is one of the most important parts of both computer science and -programming, so let's take it a step further and get some more practice by -implementing numbers. - -### What is a Number? - -For this blog post we're only going to concern ourselves with non-negative -integers. Feel free to try extending all this to include negative integers if -you want more. - -How can we represent a number? Well we could obviously use Javascript numbers -like `14`, but that's not very fun, and we're trying to minimize the number of -language features we use. - -One way to represent a number is a list whose length is the number. So we could -say that `[1, 1, 1]` means "three", `["cats", null]` means "two", and `[]` means -"zero". - -The elements themselves don't really matter, so let's just pick something we -already have: the empty list! Let's write out a few to get a feel for this: - - :::javascript - var zero = empty_list; - // [] - - var one = prepend(empty_list, empty_list); - // [ [] ] - - var two = prepend(empty_list, prepend(empty_list, empty_list)); - // [ [], [] ] - -### inc, dec - -We're going to want to *do* things with our numbers, so let's start writing -things that work with this "list of things" representation of numbers. - -Our basic building blocks are going to be `inc` and `dec` (increment and -decrement). - - :::javascript - var inc = function(n) { - return prepend(empty_list, n); - }; - - var dec = function(n) { - return tail(n); - }; - -To add 1 to a number, we just push another element on the list. So -`inc(inc(zero))` means "two". - -To subtract 1, we just pop off one of the elements: `dec(two)` means "one" -(remember we're ignoring negative numbers). - -### is\_zero - -When we started working with lists we used `is_empty` a lot, so it's probably -a good idea to create an `is_zero` function at this point: - - :::javascript - var is_zero = function(n) { - return is_empty(n); - }; - -Zero is just represented by the empty list, so this one is easy! - -### add - -Adding one is easy, but we're probably going to want to add arbitrary numbers -together. Now that we have `inc` and `dec` this is actually pretty easy: - - :::javascript - var add = function(a, b) { - if (is_zero(b)) { - return a; - } else { - return add(inc(a), dec(b)); - } - }; - -This is another recursive definition. When adding two numbers, there are two -possibilities: - -* If `b` is zero, then anything plus zero is itself, so we can just return `a`. -* Otherwise, adding `a + b` is the same as adding `(a + 1) + (b - 1)`. - -Eventually `b` will "bottom out" and return `a` (which has been steadily getting -bigger as `b` got smaller). - -Notice how we didn't say anything about lists here! The "numbers are lists -under the hood" idea has been encapsulated behind `is_zero`, `inc`, and `dec`, -so we can ignore it and work at the "number" level of abstraction from here on -out. - -### sub - -Subtraction is similar to addition, but instead of *increasing* `a` as `b` gets -smaller, we *decrease* them both together: - - :::javascript - var sub = function(a, b) { - if (is_zero(b)) { - return a; - } else { - return sub(dec(a), dec(b)); - } - }; - -Now we can say something like `add(two, sub(three, two))` and the result will be -a representation of "three" in our system (which, of course, is a list of three -elements). - -Pause for a minute now and remember that underneath numbers are lists, and -underneath lists there's nothing but functions. We can add and subtract -integers and underneath it all it's just functions shuffling around, expanding -into other functions and contracting as they're called, and this writhing mass -of lambdas somehow ends up representing `1 + 1 = 2`. That's pretty cool! - -### mul, pow - -For practice let's create a way to multiply numbers: - - :::javascript - var mul = function(a, b) { - if (is_zero(b)) { - return zero; - } else { - return add(a, mul(a, dec(b))); - } - }; - -Building on `add` makes this pretty easy. `3 * 4` is the same as `3 -+ 3 + 3 + 3 + 0`. Trace out the execution on paper if things are starting to -get away from you. Carry on when you're ready. - -`pow` ("power" or exponential) follows a similar structure as `mul`, but instead -of adding together the copies we multiply them, and our base is one instead of -zero: - - :::javascript - var pow = function(a, b) { - if (is_zero(b)) { - return one; - } else { - return mul(a, pow(a, dec(b))); - } - }; - -### is\_equal - -A common thing to do with numbers is to check if two are equal, so let's write -that: - - :::javascript - var is_equal = function(n, m) { - if (and(is_zero(n), is_zero(m))) { - return true; - } else if (or(is_zero(n), is_zero(m))) { - return false; - } else { - return is_equal(dec(n), dec(m)); - } - }; - -There are three cases here: - -* If both numbers are zero, they are equal. -* If only one number is zero (but not both, or the first case would have caught - it), then they are *not* equal. -* Otherwise, subtract one from each and try again. - -When calling this function with two non-zero numbers, both will be decremented -in tandem until one of them bottoms out at zero first, or until they bottom out -at the same time. - -### less\_than, greater\_than - -We can take a similar approach to implementing `less_than`: - - :::javascript - var less_than = function(a, b) { - if (and(is_zero(a), is_zero(b))) { - return false; - } else if (is_zero(a)) { - return true; - } else if (is_zero(b)) { - return false; - } else { - return less_than(dec(a), dec(b)); - } - }; - -The difference here is that we have four cases. - -* If both numbers are zero, then `a` is not less than `b`. -* Otherwise if `a` is zero (and we know `b` isn't) then yes, `a` is less than - `b`. -* Otherwise if `b` is zero (and we know that `a` isn't) then no, `a` cannot be - less than `b` (remember that we're ignoring negative numbers). -* Otherwise decrement both and try again. - -Once again, both numbers race to bottom out, and the outcome is decided by which -one bottoms out first. - -We could do something similar for `greater_than`, but let's do it the easy way -instead: - - :::javascript - var greater_than = function(a, b) { - return less_than(b, a); - }; - -### div, mod - -Once we have `less_than` we're ready to implement division and remainders: - - :::javascript - var div = function(a, b) { - if (less_than(a, b)) { - return zero; - } else { - return inc(div(sub(a, b), b)); - } - }; - - var rem = function(a, b) { - if (less_than(a, b)) { - return a; - } else { - return rem(sub(a, b), b); - } - }; - -This pair is a bit more complicated than the three other basic operations -because we can't deal with negative numbers. Make sure you understand how it -works. - -Full Circle ------------ - -At this point, we have a (very basic) working system of numbers built on top of -lists. Let's chase our tails a bit and implement a few more list functions that -use numbers. - -### nth - -To get the Nth item in a list, we just pop things off of it as we decrement -N until we hit zero: - - :::javascript - var nth = function(l, n) { - if (is_zero(n)) { - return head(l); - } else { - return nth(tail(l), dec(n)); - } - }; - -Under the hood there are really *two* lists getting things popped off as we -iterate, because `n` is a number, which is a list, and `dec` pops things off. -But it's much easier to read when we've abstracted away the representation of -numbers, don't you think? - -### drop, take - -Two handy functions for working with lists are `drop` and `take`. - -`drop(l, three)` will return the list with the first three elements removed. - -`take(l, three)` will return the list containing only the first three elements. - - :::javascript - var drop = function(l, n) { - if (is_zero(n)) { - return l; - } else { - return drop(tail(l), dec(n)); - } - }; - - var take = function(l, n) { - if (is_zero(n)) { - return empty_list; - } else { - return prepend(head(l), take(tail(l), dec(n))); - } - }; - -### slice - -Slicing a list is easy now that we have `drop`, `take`, and the ability to -subtract numbers: - - :::javascript - var slice = function(l, start, end) { - return take(drop(l, start), sub(end, start)); - }; - -First we drop up to the start, then take enough to get us to the end. - -### length - -We can define `length` recursively like everything else: - - :::javascript - var length = function(l) { - if (is_empty(l)) { - return zero; - } else { - return inc(length(tail(l))); - } - }; - -The length of the empty list is zero, and the length of any non-empty list is -one plus the length of its tail. - -If your mind isn't in knots by this point, consider the following: - -* Lists are made of functions. -* Numbers are made of lists whose length represents the number. -* `length` is a function that takes a list (which is a function) and returns - the length as a number (a list whose length represents the number). -* We only just now got around to defining `length` even though we've been using - numbers (which use the *length* of a list to represent a number) for a while - now! - -Are you dizzy yet? If not: - - :::javascript - var mylist = prepend(empty_list, - prepend(empty_list, - empty_list)); - var mylistlength = length(mylist); - -`mylist` is a list of two empty lists. - -`mylistlength` is the length of `mylist`... -which is "two"... -which is represented by a list of two empty lists... -which is `mylist` itself! - -Conclusion ----------- - -If you liked this twisty little story, I highly recommend you check out [The -Little Schemer][]. It was one of the first books that really changed how -I thought about programming. Don't be put off by the fact that it uses Scheme --- the language doesn't really matter. - -I've also created [a gist][] with all the code. Feel free to fork it and use it -for practice. - -You could add some more utility functions for practice writing recursively: - -* `append` to add an item to the end of a list. -* `concat` to concatenate two lists. -* `min` and `max` which take two numbers and return the minimum/maximum one. -* `remove`, which is like filter except it only leaves the elements that return - `false` for the predicate. -* `contains_number`, which checks if a specific number is inside a list of - numbers. - -Or if you want something more challenging, try implementing bigger concepts on -top of the current ones: - -* Negative numbers. -* Non-negative rational numbers. -* Negative rational numbers. -* Association lists (a data structure that associates keys with values). - -Remember: the point is not to create something that runs well on a physical -computer. Instead of thinking about how to make a particular combination of -transistors and circuits have the right voltages, think about "computing" in the -beautiful, perfect, abstract sense. - -[a gist]: https://gist.github.com/sjl/5277681 - - {% endblock article %} - diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2013/03/list-out-of-lambda.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2013/03/list-out-of-lambda.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,1097 @@ ++++ +title = "List Out of Lambda" +snip = "Down the rabbit hole we go!" +date = 2013-03-30T14:00:00Z +draft = false + ++++ + +If you ignore the practical issues of computers like size, weight, cost, heat, +and so on, what do you *really* need in a programming language? Let's play +around with this question. + +To understand this post you'll need a basic understanding of how functions in +Javascript work. If you can look at this code and understand what it prints, +you're good to go: + +```javascript +var x = 10; + +var f = function(y) { + console.log(x); + console.log(y); +} + +var g = f; + +f(1); +g(2); +``` + +This blog post is a thought exercise. It's not something you'd ever use for +real code. But just like a guitarist practices scales that she won't ever play +in a song, we programmers should be exercising our brains every so often. + +I'm going to use Javascript for the examples. Any language with first class +functions and lexical scoping (basically: closures) will work. The examples +would be prettier in a Lisp, but some people would be turned off by the syntax +and miss out on some interesting ideas. Feel free to port the code if it +bothers you. + +If you've already seen this kind of thing before (maybe you've gone through [The +Little Schemer][] or [SICP][]) you may want to just skim the code here and look +for anything new. + +If you *haven't* seen anything like this, then you're in for a treat! It's all +going to look extremely weird the first time you see it. Go slowly and make +sure you understand each piece fully before moving on to the next. These +concepts may be unintuitive, but they're built from very simple pieces. + +Finally: if you get stuck, don't worry. Tracing out the execution of a function +on paper can be a good way to wrap your brain around it (I recommend investing +in a good lap desk for comfy reading and writing). If that doesn't work, just +close the window and come back tomorrow. Sometimes new concepts need a while to +rattle around in your brain before they click into place. + +[The Little Schemer]: http://www.amazon.com/dp/0262560992/?tag=stelos-20 +[SICP]: http://www.amazon.com/dp/0070004846/?tag=stelos-20 + +{{% toc %}} + +Lists +----- + +Let's get started. One of the most common things we do as programmers is +grouping data together. Javascript has "arrays" built in to the language for +this: + +```javascript +var names = ["Alice", "Bob", "Candice"]; +``` + +What if Javascript didn't come with arrays included? Could we create them (or +something like them) ourselves? + +To answer this, let's think about the bare minimum we'd need to "bootstrap" +something like an array. There are a number of ways to do this, but we're going +to look at one in particular. + +We'll call our array-like thing a "list". To make it work, we need four parts: + +* The concept of "the empty list". +* A way to add an element to the front of a list. +* A way to take a list and get the first element. +* A way to take a list and get everything *but* the first element. + +If we have those four things, we can build on top of them to do anything else we +might want. For example: to make a list of one item, you add that item to the +front of the empty list. + +Let's narrow this down further. There are lots of ways you could implement +those four things — I'm going to use functions. Let's sketch out an outline: + +```javascript +var empty_list = null; + +var prepend = function(el, list) { + // ... +}; + +var head = function(list) { + // ... +}; + +var tail = function(list) { + // ... +}; + +var is_empty = function(list) { + // ... +}; +``` + +Here are the descriptions of each of these items. + +The `empty_list` is a special value that represents a list of zero elements. +It can be anything, so for now we'll use `null` (we'll revisit this later). + +`prepend(1, some_list)` will return a new list that looks like the old one, but +with `1` stuck onto the front of it. So if we want to create a list of the +numbers `1` and `2` we can say `prepend(1, prepend(2, empty_list))` or "prepend +one to the result of prepending 2 to the empty list". + +`head(some_list)` will return the first element in the list. Calling it on the +empty list will be undefined, so we'll just be very careful not to do that! + +`tail(some_list)` will return a new list that's like the one we gave it, but +with the first element removed. Again, calling this on an empty list will make +things explode. + +`is_empty(some_list)` will return `true` if the list given to it is the empty +list, and `false` otherwise. + +Once we have those four functions (plus the special empty list value) we can +start building on top of them, so let's figure out how to make them! + +### List Out of If + +If you haven't seen anything like this before, you might think it's time to +start creating Javascript Objects. That's certainly one way to do it. + +Since this post is a thought experiment in what we actually *need*, though, +let's try to avoid using big language features (like Objects) unless we +absolutely can't avoid it. + +So if we don't want to use other language features yet, what are we left with? +Well so far our skeleton only has functions (and `null`), so let's try those! + +Here's the first working revision of the building blocks of lists: + +```javascript +var empty_list = null; + +var prepend = function(el, list) { + return function(command) { + if (command === "head") { + return el; + } else if (command === "tail") { + return list; + } + } +}; + +var head = function(list) { + return list("head"); +}; + +var tail = function(list) { + return list("tail"); +}; + +var is_empty = function(list) { + return list === null; +}; +``` + +Go ahead and paste that into a browser console and play with it: + +```javascript +var e = empty_list; + +console.log(is_empty(e)); +// true + +var names = prepend("Alice", + prepend("Bob", + prepend("Candice", + empty_list))); + +console.log(is_empty(names)); +// False + +console.log(head(names)); +// Alice + +console.log(tail(names)); +// Some function representing the list of ("Bob", "Candice") + +console.log(head(tail(names))); +// Bob +``` + +### But Where is the Data? + +Did the definitions of those functions surprise you? Lists seem like such an +important, object-oriented concept, but there only appear to be functions here! + +Let's look at how this actually works. First of all, the "empty list" concept +is pretty straightforward: + +```javascript +var empty_list = null; + +var is_empty = function(list) { + return list === null; +}; +``` + +We could have picked any arbitrary value here. `null` seemed appropriate, so +I used that. + +Now on to the meat of things: `prepend`. + +```javascript +var prepend = function(el, list) { + return function(command) { + if (command === "head") { + return el; + } else if (command === "tail") { + return list; + } + } +}; +``` + +This is where the real magic happens. Let's think through it. + +First of all, we know that when you prepend something to a list, you're going to +get a (new) list back. So whatever `prepend` returns must be a list. + +Looking at the code, we can see it returns a function. So in our little thought +experiment, a list is actually a Javascript function under the hood! + +So what do we need to do with lists (aside from empty checking, which we've +already covered)? Well, we need to be able to get the head and the tail. When +we call `prepend(h, t)`, we happen to be conveniently specifying the head and +tail as the arguments! So in `prepend` we return a function that knows how to +return its own head or tail when asked. + +So a "list" is "a function that knows how to return its own head or tail when +asked". So our `head` and `tail` functions just need to ask nicely! + +```javascript +var head = function(list) { + return list("head"); +}; + +var tail = function(list) { + return list("tail"); +}; +``` + +That's it! We've created lists in 23 lines of code without using any fancy +things like Objects. Before you move on, make sure you really understand why +this works. Write out a few examples on paper. + +```javascript +var empty_list = null; + +var prepend = function(el, list) { + return function(command) { + if (command === "head") { + return el; + } else if (command === "tail") { + return list; + } + } +}; + +var head = function(list) { + return list("head"); +}; + +var tail = function(list) { + return list("tail"); +}; + +var is_empty = function(list) { + return list === null; +}; +``` + +### Building on the Foundations + +Now that we have lists, let's implement a few common things on top of them as +practice. + +#### map + +A common thing to do to a list is to create a new list by looping through it and +doing something to each item. This is called "map": + +```javascript +var map = function(fn, l) { + if (is_empty(l)) { + return empty_list; + } else { + return prepend(fn(head(l)), map(fn, tail(l))); + } +}; +``` + +If you're not used to recursive definitions like this, you may way to take +a few minutes and try to work out how it works. Here's an example: + +```javascript +var square = function(x) { + return x * x; +} +var numbers = prepend(1, prepend(2, prepend(3, empty_list))); + +var squared_numbers = map(square, numbers); +// map(square, [1, 2, 3]) +// prepend(square(1), map(square, [1, 2, 3])) +// prepend(square(1), prepend(square(2), map(square, [3]))) +// prepend(square(1), prepend(square(2), prepend(square(3), map(square, [])))) +// prepend(square(1), prepend(square(2), prepend(square(3), []))) +// prepend(square(1), prepend(square(2), prepend(9, []))) +// prepend(square(1), prepend(square(2), [9])) +// prepend(square(1), prepend(4, [9])) +// prepend(square(1), [4, 9]) +// prepend(1, [4, 9]) +// [1, 4, 9] +``` + +I'm using brackets here to represent lists, but remember that these aren't +arrays, but are actually the functions that were returned by `prepend`. + +If you're still not sure about this, trace out every step of `map(square, +empty_list)` on paper. Then trace out every step of `map(square, prepend(10, +empty_list))`. + +Thinking recursively like this takes some practice. I have notebooks filled +with [pages like this](http://i.imgur.com/kqu5jy9.jpg). Experienced guitarists +practice new material slowly and methodically — there's no reason programmers +shouldn't do the same. Watching the function calls expand and contract on paper +can help you feel in your gut how these things work in a way that just staring +at the words can't. + +#### filter + +We're going to start moving a bit faster now, but you should still make sure you +understand everything completely before moving on. Take as much time as you +need. Write things out. Run them. Get a feel for them. + +The next "utility" function we'll build on top of lists is `filter`, which +takes a function and a list, and returns a new list whose elements are those in +the original that make the function return `true`. Here's an example: + +```javascript +var numbers = prepend(1, prepend(2, prepend(3, empty_list))); +var is_odd = function(x) { + return x % 2 === 1; +} + +filter(is_odd, numbers); +// [1, 3] +``` + +Now let's implement `filter`: + +```javascript +var filter = function(fn, l) { + if (is_empty(l)) { + return empty_list; + } else if (fn(head(l))) { + return prepend(head(l), filter(fn, tail(l))); + } else { + return filter(fn, tail(l)); + } +}; +``` + +Take your time. Trace out some examples. Move on when you feel it in your gut. + +#### and, or, not + +Let's take a slight detour to implement a few "helper" functions. Τhese don't +have anything specifically to do with lists, but we'll need them later. + +```javascript +var not = function(x) { + if (x) { + return false; + } else { + return true; + } +}; + +var and = function(a, b) { + if (a) { + if (b) { + return true; + } else { + return false; + } + } else { + return false; + } +}; + +var or = function(a, b) { + if (a) { + return true; + } else { + if (b) { + return true; + } else { + return false; + } + } +}; +``` + +Javascript already has these things built in as `!`, `&&`, and `||`, of course, +but remember that in this thought exercise we're trying to avoid using extra +language features if we don't need them. How far can we scrape by on just +functions and `if` statements? + +One small note: these functions are just normal Javascript functions, which +means that `and(a, b)` will *not* short-circuit like `a && b` would. For our +purposes here that won't hurt us, but it's something to be aware of. + +### List Out of Lambda + +Now that we've had a bit more practice, let's go back to our definition of +lists: + +```javascript +var empty_list = null; + +var prepend = function(el, list) { + return function(command) { + if (command === "head") { + return el; + } else if (command === "tail") { + return list; + } + } +}; + +var head = function(list) { + return list("head"); +}; + +var tail = function(list) { + return list("tail"); +}; + +var is_empty = function(list) { + return list === null; +}; +``` + +There are a few things about this implementation that bother me. Our goal is to +use as few language features as possible, but we've actually used quite a few! +I count at least five: + +* Functions +* `if` statements +* Strings +* Booleans (the `true`/`false` result of `is_empty`) +* Equality checking (the `===` checks) + +It turns out we can remove most of those things at the cost of a bit of +readability (and more bending of our minds). + +Let's start by rewriting the core three functions to ditch those ugly strings, +equality checks, and even the `if` statement: + +```javascript +var prepend = function(el, list) { + return function(selector) { + return selector(el, list); + }; +}; + +var head = function(list) { + return list(function(h, t) { + return h; + }); +}; + +var tail = function(list) { + return list(function(h, t) { + return t; + }); +}; +``` + +You may want to get a snack before wrapping your brain around this one! There's +no strings, no equality checking, no `if` statements. But we still have lists! + +The `prepend` functions still returns a function, just like before. Remember +that in the last implementation, a "list" was really a function that knew how to +give out its head or its tail when asked for them. + +This time, we're inverting the "asking". In this version, a "list" is "a +function that will tell another function about both its head *and* its tail when +asked". This time the *asker* gets *both* pieces, and can decide which one they +want to use. + +Let's look at the `head` function: + +* `head` takes a list and says `return list(...)`, which means: "Hey list, + I would like you to tell all of your info to this little helper function I'm + giving you". +* The list says `return ...(el, list)`, which means: "Okay helper function, + here's my head and my tail, enjoy!" +* The helper function that `head` originally gave was `function(h, t) { return + h; }`. So when the list calls it with the head and the tail as arguments, it + returns the head and ignores the tail. +* `head` takes that result and just returns it straight through back to the + caller. + +`tail` works exactly the same way, but its helper function returns the second +argument (the tail) instead of the first. + +That's it! The equality checking and `if` statements have disappeared. Can you +describe where they've gone? What has taken their place? + +Before we move on, let's clean up the idea of the empty list. It's still using +`null` and equality checking. Let's remove those and make things a little more +uniform. + +To do this we'll need to change the other three functions a bit as well, but if +you've understood everything so far it shouldn't be too bad. + +```javascript +var empty_list = function(selector) { + return selector(undefined, undefined, true); +}; + +var prepend = function(el, list) { + return function(selector) { + return selector(el, list, false); + }; +}; + +var head = function(list) { + return list(function(h, t, e) { + return h; + }); +}; + +var tail = function(list) { + return list(function(h, t, e) { + return t; + }); +}; + +var is_empty = function(list) { + return list(function(h, t, e) { + return e; + }); +}; +``` + +We've now made lists a bit smarter. In addition to telling the helper function +their head and tail, they also tell it "am I the empty list?". We've modified +the helpers `head` and `tail` to accept (and ignore) this extra argument. + +We then modified `is_empty` to work just like `head` and `tail`. + +Finally, we've redefined `empty_list` to match the rest of the lists instead of +being a special, magic value. The empty list is now just like a normal one: +it's a function that takes an "asker" and tells that asker "Hey, my head and +tail are undefined and I am the empty list". + +I used `undefined` here which is technically another language feature because +it's easier to read. Feel free to replace it with anything you want to make it +more pure. Since we're being very careful to never call `head` or `tail` on the +empty list those values will never be seen anyway. + +So after all that, we've finally implemented the building blocks of lists with +only two things: + +* Functions. +* `true` and `false` for empty lists. + +If you're up for a challenge, think about whether you could remove that second +item (and if so, are you *really* removing it, or just using certain features of +Javascript implicitly instead of explicitly?). + +A Brief Intermission +-------------------- + +Let's take a minute to reflect on all the code we've seen so far. First, we +have an implementation of lists that uses only functions and booleans: + +```javascript +var empty_list = function(selector) { + return selector(undefined, undefined, true); +}; + +var prepend = function(el, list) { + return function(selector) { + return selector(el, list, false); + }; +}; + +var head = function(list) { + return list(function(h, t, e) { + return h; + }); +}; + +var tail = function(list) { + return list(function(h, t, e) { + return t; + }); +}; + +var is_empty = function(list) { + return list(function(h, t, e) { + return e; + }); +}; +``` + +From this point on, we can now ignore the details of how lists are implemented. +As long as we have `head`, `tail`, and `prepend` we don't need to worry about +what lists actually *are* under the hood. + +We also built a few helper functions on top of this foundation: + +```javascript +var not = function(x) { + if (x) { + return false; + } else { + return true; + } +}; + +var and = function(a, b) { + if (a) { + if (b) { + return true; + } else { + return false; + } + } else { + return false; + } +}; + +var or = function(a, b) { + if (a) { + return true; + } else { + if (b) { + return true; + } else { + return false; + } + } +}; + +var map = function(fn, l) { + if (is_empty(l)) { + return empty_list; + } else { + return prepend(fn(head(l)), map(fn, tail(l))); + } +}; + +var filter = function(fn, l) { + if (is_empty(l)) { + return empty_list; + } else if (fn(head(l))) { + return prepend(head(l), filter(fn, tail(l))); + } else { + return filter(fn, tail(l)); + } +}; +``` + +Before you move on, make sure all of this code is crystal clear. Come back +tomorrow if you need to let it sink in. We're about to go a lot deeper into the +rabbit hole, so make sure you're ready. + +Numbers +------- + +If you look at the definitions of `prepend`, `head`, and `tail`, they're pretty +mind-bending. However, the definitions of `map` and `filter` are much more +straightforward. + +This is because we encapsulated the implementation of lists into the first four +functions. We did all the hard work of building lists out of almost nothing at +all and hid it behind that simple `prepend`, `head`, and `tail` interface. + +The idea of creating things from simple pieces and abstracting them into "black +boxes" is one of the most important parts of both computer science and +programming, so let's take it a step further and get some more practice by +implementing numbers. + +### What is a Number? + +For this blog post we're only going to concern ourselves with non-negative +integers. Feel free to try extending all this to include negative integers if +you want more. + +How can we represent a number? Well we could obviously use Javascript numbers +like `14`, but that's not very fun, and we're trying to minimize the number of +language features we use. + +One way to represent a number is a list whose length is the number. So we could +say that `[1, 1, 1]` means "three", `["cats", null]` means "two", and `[]` means +"zero". + +The elements themselves don't really matter, so let's just pick something we +already have: the empty list! Let's write out a few to get a feel for this: + +```javascript +var zero = empty_list; +// [] + +var one = prepend(empty_list, empty_list); +// [ [] ] + +var two = prepend(empty_list, prepend(empty_list, empty_list)); +// [ [], [] ] +``` + +### inc, dec + +We're going to want to *do* things with our numbers, so let's start writing +things that work with this "list of things" representation of numbers. + +Our basic building blocks are going to be `inc` and `dec` (increment and +decrement). + +```javascript +var inc = function(n) { + return prepend(empty_list, n); +}; + +var dec = function(n) { + return tail(n); +}; +``` + +To add 1 to a number, we just push another element on the list. So +`inc(inc(zero))` means "two". + +To subtract 1, we just pop off one of the elements: `dec(two)` means "one" +(remember we're ignoring negative numbers). + +### is\_zero + +When we started working with lists we used `is_empty` a lot, so it's probably +a good idea to create an `is_zero` function at this point: + +```javascript +var is_zero = function(n) { + return is_empty(n); +}; +``` + +Zero is just represented by the empty list, so this one is easy! + +### add + +Adding one is easy, but we're probably going to want to add arbitrary numbers +together. Now that we have `inc` and `dec` this is actually pretty easy: + +```javascript +var add = function(a, b) { + if (is_zero(b)) { + return a; + } else { + return add(inc(a), dec(b)); + } +}; +``` + +This is another recursive definition. When adding two numbers, there are two +possibilities: + +* If `b` is zero, then anything plus zero is itself, so we can just return `a`. +* Otherwise, adding `a + b` is the same as adding `(a + 1) + (b - 1)`. + +Eventually `b` will "bottom out" and return `a` (which has been steadily getting +bigger as `b` got smaller). + +Notice how we didn't say anything about lists here! The "numbers are lists +under the hood" idea has been encapsulated behind `is_zero`, `inc`, and `dec`, +so we can ignore it and work at the "number" level of abstraction from here on +out. + +### sub + +Subtraction is similar to addition, but instead of *increasing* `a` as `b` gets +smaller, we *decrease* them both together: + +```javascript +var sub = function(a, b) { + if (is_zero(b)) { + return a; + } else { + return sub(dec(a), dec(b)); + } +}; +``` + +Now we can say something like `add(two, sub(three, two))` and the result will be +a representation of "three" in our system (which, of course, is a list of three +elements). + +Pause for a minute now and remember that underneath numbers are lists, and +underneath lists there's nothing but functions. We can add and subtract +integers and underneath it all it's just functions shuffling around, expanding +into other functions and contracting as they're called, and this writhing mass +of lambdas somehow ends up representing `1 + 1 = 2`. That's pretty cool! + +### mul, pow + +For practice let's create a way to multiply numbers: + +```javascript +var mul = function(a, b) { + if (is_zero(b)) { + return zero; + } else { + return add(a, mul(a, dec(b))); + } +}; +``` + +Building on `add` makes this pretty easy. `3 * 4` is the same as `3 ++ 3 + 3 + 3 + 0`. Trace out the execution on paper if things are starting to +get away from you. Carry on when you're ready. + +`pow` ("power" or exponential) follows a similar structure as `mul`, but instead +of adding together the copies we multiply them, and our base is one instead of +zero: + +```javascript +var pow = function(a, b) { + if (is_zero(b)) { + return one; + } else { + return mul(a, pow(a, dec(b))); + } +}; +``` + +### is\_equal + +A common thing to do with numbers is to check if two are equal, so let's write +that: + +```javascript +var is_equal = function(n, m) { + if (and(is_zero(n), is_zero(m))) { + return true; + } else if (or(is_zero(n), is_zero(m))) { + return false; + } else { + return is_equal(dec(n), dec(m)); + } +}; +``` + +There are three cases here: + +* If both numbers are zero, they are equal. +* If only one number is zero (but not both, or the first case would have caught + it), then they are *not* equal. +* Otherwise, subtract one from each and try again. + +When calling this function with two non-zero numbers, both will be decremented +in tandem until one of them bottoms out at zero first, or until they bottom out +at the same time. + +### less\_than, greater\_than + +We can take a similar approach to implementing `less_than`: + +```javascript +var less_than = function(a, b) { + if (and(is_zero(a), is_zero(b))) { + return false; + } else if (is_zero(a)) { + return true; + } else if (is_zero(b)) { + return false; + } else { + return less_than(dec(a), dec(b)); + } +}; +``` + +The difference here is that we have four cases. + +* If both numbers are zero, then `a` is not less than `b`. +* Otherwise if `a` is zero (and we know `b` isn't) then yes, `a` is less than + `b`. +* Otherwise if `b` is zero (and we know that `a` isn't) then no, `a` cannot be + less than `b` (remember that we're ignoring negative numbers). +* Otherwise decrement both and try again. + +Once again, both numbers race to bottom out, and the outcome is decided by which +one bottoms out first. + +We could do something similar for `greater_than`, but let's do it the easy way +instead: + +```javascript +var greater_than = function(a, b) { + return less_than(b, a); +}; +``` + +### div, mod + +Once we have `less_than` we're ready to implement division and remainders: + +```javascript +var div = function(a, b) { + if (less_than(a, b)) { + return zero; + } else { + return inc(div(sub(a, b), b)); + } +}; + +var rem = function(a, b) { + if (less_than(a, b)) { + return a; + } else { + return rem(sub(a, b), b); + } +}; +``` + +This pair is a bit more complicated than the three other basic operations +because we can't deal with negative numbers. Make sure you understand how it +works. + +Full Circle +----------- + +At this point, we have a (very basic) working system of numbers built on top of +lists. Let's chase our tails a bit and implement a few more list functions that +use numbers. + +### nth + +To get the Nth item in a list, we just pop things off of it as we decrement +N until we hit zero: + +```javascript +var nth = function(l, n) { + if (is_zero(n)) { + return head(l); + } else { + return nth(tail(l), dec(n)); + } +}; +``` + +Under the hood there are really *two* lists getting things popped off as we +iterate, because `n` is a number, which is a list, and `dec` pops things off. +But it's much easier to read when we've abstracted away the representation of +numbers, don't you think? + +### drop, take + +Two handy functions for working with lists are `drop` and `take`. + +`drop(l, three)` will return the list with the first three elements removed. + +`take(l, three)` will return the list containing only the first three elements. + +```javascript +var drop = function(l, n) { + if (is_zero(n)) { + return l; + } else { + return drop(tail(l), dec(n)); + } +}; + +var take = function(l, n) { + if (is_zero(n)) { + return empty_list; + } else { + return prepend(head(l), take(tail(l), dec(n))); + } +}; +``` + +### slice + +Slicing a list is easy now that we have `drop`, `take`, and the ability to +subtract numbers: + +```javascript +var slice = function(l, start, end) { + return take(drop(l, start), sub(end, start)); +}; +``` + +First we drop up to the start, then take enough to get us to the end. + +### length + +We can define `length` recursively like everything else: + +```javascript +var length = function(l) { + if (is_empty(l)) { + return zero; + } else { + return inc(length(tail(l))); + } +}; +``` + +The length of the empty list is zero, and the length of any non-empty list is +one plus the length of its tail. + +If your mind isn't in knots by this point, consider the following: + +* Lists are made of functions. +* Numbers are made of lists whose length represents the number. +* `length` is a function that takes a list (which is a function) and returns + the length as a number (a list whose length represents the number). +* We only just now got around to defining `length` even though we've been using + numbers (which use the *length* of a list to represent a number) for a while + now! + +Are you dizzy yet? If not: + +```javascript +var mylist = prepend(empty_list, + prepend(empty_list, + empty_list)); +var mylistlength = length(mylist); +``` + +`mylist` is a list of two empty lists. + +`mylistlength` is the length of `mylist`... +which is "two"... +which is represented by a list of two empty lists... +which is `mylist` itself! + +Conclusion +---------- + +If you liked this twisty little story, I highly recommend you check out [The +Little Schemer][]. It was one of the first books that really changed how +I thought about programming. Don't be put off by the fact that it uses Scheme +-- the language doesn't really matter. + +I've also created [a gist][] with all the code. Feel free to fork it and use it +for practice. + +You could add some more utility functions for practice writing recursively: + +* `append` to add an item to the end of a list. +* `concat` to concatenate two lists. +* `min` and `max` which take two numbers and return the minimum/maximum one. +* `remove`, which is like filter except it only leaves the elements that return + `false` for the predicate. +* `contains_number`, which checks if a specific number is inside a list of + numbers. + +Or if you want something more challenging, try implementing bigger concepts on +top of the current ones: + +* Negative numbers. +* Non-negative rational numbers. +* Negative rational numbers. +* Association lists (a data structure that associates keys with values). + +Remember: the point is not to create something that runs well on a physical +computer. Instead of thinking about how to make a particular combination of +transistors and circuits have the right voltages, think about "computing" in the +beautiful, perfect, abstract sense. + +[a gist]: https://gist.github.com/sjl/5277681 + + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2013/04/git-koans.html --- a/content/blog/2013/04/git-koans.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,167 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "Git Koans" - snip: "The path to enlightenment is long and arduous." - created: 2013-04-08 10:16:00 - %} - - {% block article %} - -Inspired by [Vim Koans][]. - -[Vim Koans]: http://blog.sanctum.geek.nz/vim-koans/ - -[TOC] - -Silence -------- - -A Python programmer handed her `~/.gitconfig` to Master Git. Among the many -lines were the following: - - :::text - [alias] - ; Explicit is better than implicit. If we want to merge - ; we should do so ourselves. - pull = pull --ff-only - -Master Git nodded. "`git pull origin master`," said the programmer. - -Master Git pulled down the latest changes on `master` and automatically merged -them with the programmer's changes. - -"But Master Git, did I not say to only fast-forward in my configuration?!" she -cried. - -Master Git looked at her, nodded, and said nothing. - -"Then why did you not warn me of a problem with my configuration?" she asked. - -Master Git replied: "there was no problem." - -Months later the programmer was reading `git --help config` for a different -reason and found enlightenment. - -One Thing Well --------------- - -A UNIX programmer was working in the cubicle farms. As she saw Master Git -traveling down the path, she ran to meet him. - -"It is an honor to meet you, Master Git!" she said. "I have been studying the -UNIX way of designing programs that each do one thing well. Surely I can learn -much from you." - -"Surely," replied Master Git. - -"How should I change to a different branch?" asked the programmer. - -"Use `git checkout`." - -"And how should I create a branch?" - -"Use `git checkout`." - -"And how should I update the contents of a single file in my working directory, -without involving branches at all?" - -"Use `git checkout`." - -After this third answer, the programmer was enlightened. - -Only the Gods -------------- - -The great historian was trying to unravel the intricacies of an incorrect merge -that had happened many months ago. He made a pilgrimage to Master Git to ask -for his help. - -"Master Git," said the historian, "what is the nature of history?" - -"History is immutable. To rewrite it later is to tamper with the very fabric of -existence." - -The historian nodded, then asked: "Is that why rebasing commits that have been -pushed is discouraged?" - -"Indeed," said Master Git. - -"Splendid!" exclaimed the historian. "I have a historical record of a merge -commit with two parents. How can I find out which branch each parent was -originally made on?" - -"History is ephemeral," replied Master Git, "the knowledge you seek can be -answered only by the gods." - -The historian hung his head as enlightenment crushed down upon him. - -The Hobgoblin -------------- - -A novice was learning at the feet of Master Git. At the end of the lesson he -looked through his notes and said, "Master, I have a few questions. May I ask -them?" - -Master Git nodded. - -"How can I view a list of *all* tags?" - -"`git tag`", replied Master Git. - -"How can I view a list of *all* remotes?" - -"`git remote -v`", replied Master Git. - -"How can I view a list of *all* branches?" - -"`git branch -a`", replied Master Git. - -"And how can I view the current branch?" - -"`git rev-parse --abbrev-ref HEAD`", replied Master Git. - -"How can I delete a remote?" - -"`git remote rm`", replied Master Git. - -"And how can I delete a branch?" - -"`git branch -d`", replied Master Git. - -The novice thought for a few moments, then asked: "Surely some of these could be -made more consistent, so as to be easier to remember in the heat of coding?" - -Master Git snapped his fingers. A hobgoblin entered the room and ate the novice -alive. In the afterlife, the novice was enlightened. - -The Long and Short of It ------------------------- - -Master Git and a novice were walking along a bridge. - -The novice, wanting to partake of Master Git's vast knowledge, said: -"`git branch --help`". - -Master Git sat down and lectured her on the seven forms of `git branch`, and -their many options. - -They resumed walking. A few minutes later they encountered an experienced -developer traveling in the opposite direction. He bowed to Master Git and said -"`git branch -h`". Master Git tersely informed him of the most common `git -branch` options. The developer thanked him and continued on his way. - -"Master," said the novice, "what is the nature of long and short options for -commands? I thought they were equivalent, but when that developer used `-h` you -said something different than when I said `--help`." - -"Perspective is everything," answered the Master. - -The novice was puzzled. She decided to experiment and said "`git -h branch`". - -Master Git turned and threw himself off the railing, falling to his death on the -rocks below. - -Upon seeing this, the novice was enlightened. - - {% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2013/04/git-koans.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2013/04/git-koans.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,165 @@ ++++ +title = "Git Koans" +snip = "The path to enlightenment is long and arduous." +date = 2013-04-08T10:16:00Z +draft = false + ++++ + +Inspired by [Vim Koans][]. + +[Vim Koans]: http://blog.sanctum.geek.nz/vim-koans/ + +{{% toc %}} + +Silence +------- + +A Python programmer handed her `~/.gitconfig` to Master Git. Among the many +lines were the following: + +```text +[alias] +; Explicit is better than implicit. If we want to merge +; we should do so ourselves. +pull = pull --ff-only +``` + +Master Git nodded. "`git pull origin master`," said the programmer. + +Master Git pulled down the latest changes on `master` and automatically merged +them with the programmer's changes. + +"But Master Git, did I not say to only fast-forward in my configuration?!" she +cried. + +Master Git looked at her, nodded, and said nothing. + +"Then why did you not warn me of a problem with my configuration?" she asked. + +Master Git replied: "there was no problem." + +Months later the programmer was reading `git --help config` for a different +reason and found enlightenment. + +One Thing Well +-------------- + +A UNIX programmer was working in the cubicle farms. As she saw Master Git +traveling down the path, she ran to meet him. + +"It is an honor to meet you, Master Git!" she said. "I have been studying the +UNIX way of designing programs that each do one thing well. Surely I can learn +much from you." + +"Surely," replied Master Git. + +"How should I change to a different branch?" asked the programmer. + +"Use `git checkout`." + +"And how should I create a branch?" + +"Use `git checkout`." + +"And how should I update the contents of a single file in my working directory, +without involving branches at all?" + +"Use `git checkout`." + +After this third answer, the programmer was enlightened. + +Only the Gods +------------- + +The great historian was trying to unravel the intricacies of an incorrect merge +that had happened many months ago. He made a pilgrimage to Master Git to ask +for his help. + +"Master Git," said the historian, "what is the nature of history?" + +"History is immutable. To rewrite it later is to tamper with the very fabric of +existence." + +The historian nodded, then asked: "Is that why rebasing commits that have been +pushed is discouraged?" + +"Indeed," said Master Git. + +"Splendid!" exclaimed the historian. "I have a historical record of a merge +commit with two parents. How can I find out which branch each parent was +originally made on?" + +"History is ephemeral," replied Master Git, "the knowledge you seek can be +answered only by the gods." + +The historian hung his head as enlightenment crushed down upon him. + +The Hobgoblin +------------- + +A novice was learning at the feet of Master Git. At the end of the lesson he +looked through his notes and said, "Master, I have a few questions. May I ask +them?" + +Master Git nodded. + +"How can I view a list of *all* tags?" + +"`git tag`", replied Master Git. + +"How can I view a list of *all* remotes?" + +"`git remote -v`", replied Master Git. + +"How can I view a list of *all* branches?" + +"`git branch -a`", replied Master Git. + +"And how can I view the current branch?" + +"`git rev-parse --abbrev-ref HEAD`", replied Master Git. + +"How can I delete a remote?" + +"`git remote rm`", replied Master Git. + +"And how can I delete a branch?" + +"`git branch -d`", replied Master Git. + +The novice thought for a few moments, then asked: "Surely some of these could be +made more consistent, so as to be easier to remember in the heat of coding?" + +Master Git snapped his fingers. A hobgoblin entered the room and ate the novice +alive. In the afterlife, the novice was enlightened. + +The Long and Short of It +------------------------ + +Master Git and a novice were walking along a bridge. + +The novice, wanting to partake of Master Git's vast knowledge, said: +"`git branch --help`". + +Master Git sat down and lectured her on the seven forms of `git branch`, and +their many options. + +They resumed walking. A few minutes later they encountered an experienced +developer traveling in the opposite direction. He bowed to Master Git and said +"`git branch -h`". Master Git tersely informed him of the most common `git +branch` options. The developer thanked him and continued on his way. + +"Master," said the novice, "what is the nature of long and short options for +commands? I thought they were equivalent, but when that developer used `-h` you +said something different than when I said `--help`." + +"Perspective is everything," answered the Master. + +The novice was puzzled. She decided to experiment and said "`git -h branch`". + +Master Git turned and threw himself off the railing, falling to his death on the +rocks below. + +Upon seeing this, the novice was enlightened. + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2013/09/teach-dont-tell.html --- a/content/blog/2013/09/teach-dont-tell.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,746 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "Teach, Don't Tell" - snip: "Documentation Matters" - created: 2013-09-03 10:55:00 - flattr: true - %} - - {% block article %} - -This post is about writing technical documentation. More specifically: it's -about writing documentation for programming languages and libraries. - -I love reading great documentation. When I have a question and the -documentations explains the answer almost as if the author has magically -anticipated my problem, I get a warm, fuzzy feeling inside. I feel a connection -with the writer that makes me smile. - -I also love writing documentation. Being able to rewire the neurons in -someone's brain so that they understand something they didn't understand before -is extremely satisfying. Seeing (or hearing about) the "click" when a bunch of -concepts suddenly fall together and make sense never fails to make my day. - -This post is going to be about what I think good documentation is and how -I think you should go about writing it. I'm not perfect, so you should take -everything with a grain of salt, but I hope you'll find it useful and -thought-provoking even if you don't agree with me about everything. - -I'd like to say thanks to [Craig Zheng][cz] and [Honza Pokorny][honza] for -proofreading this. - -[cz]: http://craigzheng.com/ -[honza]: http://honza.ca/ - -[TOC] - -## Prior Reading - -Before you read this post there are two other things I think you should read -first. - -The first is Jacob Kaplan-Moss' [Writing Great Documentation][] series. He's -certainly more qualified than I am to write about this stuff, so you should -check that out if you haven't already. A lot of what I say here is going to -agree with and build on the ideas he talked about. - -The other thing you should read is [The Science of Scientific Writing][] by -George Gopen and Judith Swan. Don't be put off by the fact that it's written -for scientists publishing papers in journals. Everything in that article -applies equally well to programmers writing technical docs. Read the entire -thing. It's worth it. - -[Writing Great Documentation]: http://jacobian.org/writing/great-documentation/ -[The Science of Scientific Writing]: http://www.americanscientist.org/issues/id.877,y.0,no.,content.true,page.1,css.print/issue.aspx - -## Why Do We Document? - -Let's get started. The first thing to nail down is *why* we're documenting -a programming language or library in the first place. There are many things you -might want to accomplish, but I'm going to boil them down into a single -statement: - -**The purpose of technical documentation is to take someone who has never seen -your project, teach them to be an expert user of it, and support them once they -become an expert.** - -At first glance this probably doesn't seem too controversial or interesting. -But there's one word in there that makes *all* the difference, and it frames my -entire perspective on documentation. - -## Teaching - -If you want to take a person who has never played the guitar and turn them into -a virtuoso guitarist, how can you do that? - -You *teach* them. - -If you want to take a high school student and turn them into a computer -scientist, how can you do that? - -You *teach* them. - -If you want to take a programmer who has never seen your library before and turn -them into an expert user of it, how can you do that? - -You *teach* them! - -Guitar lessons are usually taught in person, one-on-one, with a teacher. -Computer Science is usually taught by professors in classrooms. Programming -library usage is usually taught by documentation. - -If the goal of documentation is to turn novices into experts, then *the -documentation must teach*. You should think of your documentation as a lesson -(or series of lessons) because *that's what it is*. - -When writing technical documentation you (usually) don't have the advantage of -having a one-on-one dialog with the learners. This makes it a bit more -difficult, but not impossible as long as you're careful. Your documentation -needs to fill the role of both the in-person lessons *and* the textbook. - -The rest of this post will be almost entirely about how to apply the -"documentation is teaching" mindset to writing programming docs. - -## A Play in Seven Acts - -I'm going to break up the content of this post with some venting about *bad* -documentation. If you want to skip these little rants, go ahead. - -Each act in our play has two characters: a teenager and a parent. The teenager -has just turned sixteen and would like to learn to drive so they can hang out -with their friends without relying on their parents to drive them everywhere. - -Each act will demonstrate a caricature of a particularly *bad* form of -documentation. I hope these little metaphors will help show why certain forms -of documentation are ineffective cop-outs and why you should write *real* -documentation instead. - -## Act 1: "Read the Source" - -Our play starts with a son and father sitting at the breakfast table. The son -is munching on some cereal before school while the father reads his iPad before -leaving for work. - -The son says: "Hey Dad, you said you were going to teach me how to drive after -school today. Are we still going to do that?" - -The father, without looking up from his iPad, replies: "Of course, son. The car -is in the garage and I laid out a set of wrenches on the workbench. Take the -car apart and look at each piece, then put it back together. Once you've done -that I'll take you to the DMV for your driving test." - -The son quietly continued eating his cereal. - -If you use many open source libraries you've undoubtedly encountered some whose -README says something like "read the source". Every time I see one, I die -a little bit inside. - -Source code is *not* documentation. Can you learn to be a guitarist by simply -listening to a piece of music intently? Can you become a painter by visiting -a lot of museums? Of course not! - -Let me be clear: I'm not trying to say that reading source code isn't a valuable -thing to do. It is! - -Looking at other artists' paintings is extremely useful *once you know how to -paint*. Learning how the brakes of a car are constructed can save your life, -*once you know how to drive*. - -Once your library's users know how to work with it, reading its source is -absolutely worth their time. But you can't look at the finished product and -understand the perspective and ideas that went into it without additional -guidance. That's a job for documentation. - -## Tools of the Trade - -Writing good documentation doesn't take much in the way of tools. - -For example: you don't need a thesaurus. Don't try to avoid using the same -words by substituting synonyms. Just talk to your users like you would talk to -another human being in real life! - -If you use the same word ten times in a row your readers probably won't even -notice. But I guarantee they're going to notice if you throw in strange, -uncommon words for no good reason. - -There are two tools I *will* recommend before moving on. The first isn't -actually a tool, but a skill. - -To write great documentation, you need to be able to type. - -When you write docs you'll inevitably write yourself into a corner and realize -you need to take a new direction. If you don't type quickly, you might be -hesitant to throw away writing that doesn't really work. You need to learn to -type well so you don't feel bad throwing away a chunk of a thousand words that -don't fit. - -Steve Yegge's article [Programming's Dirty Little Secret][type-dammit] is -a great rant on this topic. - -You should also get yourself a nice keyboard. A good keyboard won't make you -a good writer (just like a good guitar won't make you a good guitarist), but it -*will* make you want to write more just for the sheer joy of using -a well-designed piece of equipment. - -I started practicing guitar a lot more after I got a new guitar that was much -nicer than my old one. If you spend a hundred dollars, get a nice keyboard, and -end up wanting to write more, it was worth it! - -Be thankful that a nice keyboard only costs $100 to $300 and not several -thousand dollars like a nice instrument. - -[type-dammit]: http://steve-yegge.blogspot.com/2008/09/programmings-dirtiest-little-secret.html - -## Act 2: "Read the Tests" - -The next scene opens with a mother picking her daughter up from high school. - -"Hi Mom", she says, "are you still going to teach me to drive today?" - -"Yep!" she replies. "Let's get going." - -After ten minutes of driving they arrive at the Chevrolet factory. - -The girl looks around, puzzled. She asks: "What are we doing here?" - -The mother smiles and says: "You're in luck, honey, my friend Jim works here at -the Chevy plant, and he's gonna let you watch a few crash tests of the new -Malibu! Once you see a few cars smash into each other, I'll take you down to -the DMV for your driving test." - -Another common form of "documentation" is the README instructing users to "read -the tests". - -Tests aren't docs. - -Again, let's be clear: once you already know how to use a library, reading the -tests is *very* useful. But you need documentation to make you an expert user -first! - -You don't learn to drive by watching crash tests. But learning how your car -behaves during a crash can save your life *once you know how to drive*. - -A common argument I see goes something like this: - -"The tests use the library, so they're a good example of how to use it!" - -This is true in some very superficial sense, but completely misses the mark. - -Most of the tests are probably going to deal with edge cases. Edge cases are -things a normal user won't be encountering very often (otherwise they wouldn't -be edge cases!). - -If you're lucky, you might get a test that verifies the library works correctly -on a normal set of input. But a "normal set of input" is what the users are -going to be working with the majority the time! - -Tests simply aren't a good guide to what a user is going to be encountering on -a day-to-day basis. They can't teach a novice to be an expert. - -## How to Teach - -If you accept my idea that the purpose of documentation is to *teach* users, the -next question is obviously: "How do I teach my users?" - -I've been lucky enough to have the chance to teach dancing semi-formally for -around 6 or 7 years, and lots of various other things informally for a long -time. The only way to *really* learn how to teach is to *do it*. - -There's no substitute for sitting down with someone face-to-face and teaching -them something. **If you want to write better documentation, you need to -practice teaching**. - -I'm not talking about writing out lesson plans or anything nearly so formal. Do -you have a hobby (not programming)? If so, spend a couple of hours on a weekend -teaching a friend about it. You'll get some practice teaching and they'll get -to learn something new. - -(If you don't have any non-programming hobbies, maybe you should find some.) - -If you like photography, teach someone the basics of exposure and composition. -If you dance, teach them some basic steps. If you play an instrument, teach -them how to play a simple song. If you like camping, teach them what all the -gear is for. You get the idea. - -Don't go overboard. You don't need to give someone a degree, you just need to -practice teaching a little bit. You need to practice the art of rewiring -someone's neurons with your words. - -Once you jump into teaching something (even something simple) you'll probably -realize that although you know how to do it yourself, it's a lot harder to teach -someone else. - -This is obvious when you're working face-to-face with someone. When you tell -them how to play a C major chord on the guitar and they only produce a strangled -squeak, it's clear that you need to slow down and talk about how to press down -on the strings properly. - -As programmers, we almost *never* get this kind of feedback about our -documentation. We don't see that the person on the other end of the wire is -hopelessly confused and blundering around because they're missing something we -thought was obvious (but wasn't). Teaching someone in person helps you learn to -anticipate this, which will pay off (for your users) when you're writing -documentation. - -With all that said, I do want to also talk a little about the actual process of -teaching. - -The best description of how to teach that I've seen so far is from the book [How -to Solve It][]. Everyone who wants to teach should read this book. The passage -that really jumped out at me is right in the first page of the first chapter: - -> The best [way for the teacher to help their student] is to help the student -> naturally. The teacher should put himself in the student's place, he should -> see the student's case, he should try to understand what is going on in the -> student's mind, and ask a question or indicate a step that *could have -> occurred to the student himself*. - -This, right here, is the core of teaching. This is it. This is how you do it. - -People don't learn by simply absorbing lots of unstructured information as it's -thrown at them. You can't read a Spanish dictionary to someone to teach them -Spanish. - -When you want to teach someone you need to put yourself in their shoes and walk -along the path with them. Hold their hand, guide them around the dangerous -obstacles and catch them when they fall. *Don't* carry them. *Certainly -don't* just drive them to the destination in your car! - -The process needs to go something like this: - -1. Figure out what they already know. -2. Figure out what you want them to know after you finish. -3. Figure out a single idea or concept that will move state 1 a little bit - closer to state 2. -4. Nudge the student in the direction of that idea. -5. Repeat until state 1 becomes state 2. - -Too often I see documentation that has very carefully considered step 2, and -then simply presents it to the reader as a pronouncement from God. That isn't -teaching. That's telling. People don't learn by being *told*, they -learn by being *taught*. - -[How to Solve It]: http://www.amazon.com/dp/069111966X/?tag=stelos-20 - -## Act 3: "Literate Programming" - -The third act opens with a daughter talking to her mother the day before her -sixteenth birthday. - -"Hey Mom," she says, "I don't know if you got me a present yet, but if not, what -I'd *really* like for my birthday are driving lessons." - -The mother smiles and says: "Don't worry, it's all taken care of. Just wait for -tomorrow." - -The next day at her birthday party she unwraps the present from her mom. Inside -is a DVD of the show How It's Made. She looks quizzically at her mother. - -"That DVD has an episode about the factory that builds your car! Once you watch -the whole thing I'll take you for your driving test." - -A horrible trend I've noticed lately is using "literate programming" tools like -Docco, Rocco, etc and telling users to read the results for documentation. - -Programming languages and libraries are tools. Knowing how a tool was made -doesn't mean you know how to use it. When you take guitar lessons, you don't -visit a luthier to watch her shape a Telecaster out of Ash wood. - -Knowing how your car was built can help you, *once you know how to drive*. - -Knowing how your guitar was built can help you, *once you know how to play*. - -A common theme throughout these acts/rants is that all of these things I'm -picking on (source, tests, literate programming, and more) are good things with -real benefits *once you have actual documentation in place to teach users*. - -But until that happens, they're actually *bad* because they let you pretend -you've written documentation and your job is done (JKM mentions this in his -series). Your job is not done until you've taught your users enough to become -experts. *Then* they can take advantage of all these extras. - -## The Anatomy of Good Documentation - -The rest of this post is going to be about the individual components that make -up good documentation. My views are pretty similar to JKM's, so if you haven't -read the series I mentioned in the first section you should probably do that. - -In my mind I divide good documentation into roughly four parts: - -1. First Contact -2. The Black Triangle -3. The Hairball -4. The Reference - -There don't necessarily have to be four separate documents for each of these. -In fact the first two can usually be combined into a single file, while the last -two should probably be split into many pieces. But I think each component is -a distinct, important part of good documentation. - -Let's take a look at each. - -## First Contact - -When you release a new programming language or library into the wild, the -initial state of your "users" is going to be blank. The things they need -to know when they encounter your library are: - -1. What is this thing? -2. Why would I care about this thing? -3. Is it worth the effort to learn this thing? - -Your "first contact" documentation should explain these things to them. - -You don't need to explain things from first principles. Try to put yourself in -the shoes of your users. When you're teaching your teenager to drive, you don't -need to explain what a "wheel" is. They probably have some experience with -"things on wheels that you move around in" like lawn mowers or golf carts (or -even video games). - -Likewise: if you're creating a web framework, most of the people that stumble on -to your project are probably going to know what "HTML" is. It's good to err -a little bit on the side of caution and explain a little more than to assume too -much, but you can be practical here. - -Your "first contact" docs should explain what, in plain words, your thing does. -It should show someone why they should care about that. Will it save them time? -Will it take more time, but be more stable in exchange? Is it just plain fun? - -For bonus points, you can also mention why someone might want to *not* use your -project. Barely anyone ever mentions the tradeoffs involved with using their -work, so to see a project do this is refreshing. - -Finally, the user needs to know if it's worth spending some of their finite -amount of time on this planet learning more about your project. You should -explicitly spell out things like: - -* What license the project uses (so they know if it's practical to use). -* Where the bug tracker is (so they can see issues). -* Where the source code is (so they can see if it's (relatively) recently - maintained). -* Where the documentation is (so they can skim it and get an idea of the effort - that's going to be involved in becoming an expert). - - - -## Act 4: "Read the Docstrings" - -Scene four. A father is finally making good on his promise to give his daughter -driving lessons. - -"Okay Dad," she says, "I'm ready. I've never driven a car before. Where do we -start?" - -A woman in her mid-forties walks through the door. "Who's this?" the daughter -asks. - -"This is your driving teacher, Ms. Smith." the father replies. "She's going to -sit in the passenger seat with you while you drive the two hour trip to visit -grandpa. If you have any questions about a part of the car while you're -driving, you can ask her and she'll tell you all about that piece. Here are -the keys, good luck!" - -In languages with [docstrings][] there's a tendency to write great docstrings -and call them documentation. I'm sure the "doc" in the word "docstrings" -contributes to this. - -Docstrings don't provide any organization or order (beyond "the namespace they -happen to be implemented in"). Users need to somehow know the name of the -function they need to even be able to *see* the docstring, and they can't know -that unless you *teach* them. - -Again, docstrings are great *once you know the project*. But when you're -teaching a novice how to use your library, you need to guide them along they -way and not sit back and answer questions when they manage to guess a magic -word correctly. - -[docstrings]: https://en.wikipedia.org/wiki/Docstring - -## The Black Triangle - -The next important piece of documentation is [the "black triangle"][]. It -should be a relatively short guide to getting your project up and running so the -user can poke at it. - -This serves a couple of purposes. First, it lets the user verify that yes, this -collection of bytes is actually going to run and *do something* on their -machine. It's a quick sanity check that the project hasn't bit rotted and is -still viable to use at that point in time. More importantly, it lets your -prospective user [get some paint on the canvas][paint]. - -Imagine if you went to your first guitar lesson and the teacher said: "Okay, -we're going to start by learning 150 different chords. Then in about six months -we can play some songs." No guitar teacher does that. They teach you three -chords and give you a couple of cheesy pop songs to play. It helps the student -get a feel for what being a guitarist as a whole is going to be like, and it -gives them something to help keep their interest. - -Your "black triangle" documentation should be a short guide that runs the user -through the process of retrieving, installing, and poking your project or -language. - -"Short" here is a relative word. Some projects are going to require more setup -to get running. If the benefits are enough to justify the effort, that's not -necessarily a problem. But try to keep this as short as possible. *Just get -something on the screen* and move on. - -[the "black triangle"]: http://rampantgames.com/blog/2004/10/black-triangle.html -[paint]: http://worrydream.com/LearnableProgramming/#react - -## Act 5: "Read the API Docs" - -Our next scene opens a year after the last, with the father from the last scene -talking to his son. - -(Sadly, the daughter in that scene died in a car crash because she didn't know -to ask Ms. Smith about seatbelts before getting on the expressway. Ms. Smith -was wearing hers, of course.) - -"Okay son, I know you're a little scared of driving because of what happened to -your sister, but I've fixed the problem." - -He hands the young man an inch-thick book. "Asking Ms. Smith questions along -the way clearly didn't work, so we had her write out a paragraph or two about -each piece of your car. Go ahead and read the entire manual cover to cover and -then drive down to see grandpa." - -API documentation is like the user's manual of a car. When something goes wrong -and you need to replace a tire it's a godsend. But if you're learning to drive -it's not going to help you because *people don't learn by reading alphabetized -lists of disconnected information*. - -If you actually try to teach someone to use your project face-to-face, you'll -probably find yourself talking about things in one namespace for a while, then -switching to another to cover something related, then switching back to the -first. Learning isn't a straight path through the alphabet, it's a zig-zaggy -ramble through someone else's brain. - -## The Hairball - -This brings me to the next type of documentation: "the hairball". By now the -user has hopefully seen the "first contact" docs and the "black triangle" docs. -You've got them hooked and ready to learn, but they're still novices. - -The "hairball" is the twisted, tangled maze of teaching that is going to take -these novices and turn them into expert users. It's going to mold their brains, -one nudge at a time, until they have a pretty good understanding of how your -project works. - -You'll usually want to organize the "hairball" into sections (unless this is -a very small project). These sections will probably *kind of* line up with -namespaces in your project's public API, but when it makes sense to deviate you -should do so. - -Don't be afraid to write. Be concise but err on the side of explaining a bit -too much. Programmers are pretty good at skimming over things they already -know, but if you forget to include a crucial connection it can leave your users -lost and stumbling around in the woods. - -You should have a table of contents that lists each section of the "hairball". -And then each section should have its own table of contents that lists the -sections inside it. A table of contents is a wonderful way to get a birds-eye -view of what you're about to read, to prepare your brain for it. And of -course it's also handy for navigating around. - -This is where your hobby-teaching practice and your reading of How to Solve It -are going to come in handy. Put yourself in a user's brain and figure out each -little connective leap they're going to need to make to become an expert. - -## Act 6: "Read the Wiki" - -In the penultimate scene, a mother has signed her teenage son up for an -after-school driving class. - -On the first day, the teacher hands them a syllabus detailing what they're going -to cover, talks about grading, and sends them home a bit early. - -On the second day, she gives them a brief overview of the various pieces of -a car and how they work together. She also talks about a few of the most -important laws they'll need to be aware of. - -On the third day, the teacher calls in sick and they have a substitute. He -covers the material for half of the fifth day in the syllabus. He has to leave -early, so he brings in his nineteen year old daughter to finish the class. -She covers the first half of the fourth day's material. - -The fourth day the students arrive to find a note on the door saying the class -has been cancelled because the teacher is still sick and they can't find -a substitute. There's a note saying "TODO: we'll talk about the material -later." - -The fifth day the teacher has partially recovered, so she returns and covers the -material for the fifth day. It's a bit hard to understand her because she's had -half a bottle of Nyquil and is slurring most of her words and keeps saying -"cat" instead of "car". - -All the students fail the driving test. - -Wikis are an abomination. They are the worst form of "documentation". - -First of all: assuming they work as intended, they have no coherent voice. - -Have you ever taken a class with multiple teachers at once? Probably not, -because it doesn't work very well (with exceptions for things like partnered -dancing where there are distinct lead/follow parts). - -Worse still: have you ever taken a class where there's one jackass in the room -who keeps constantly raising his hand and offering his own (often incorrect) -opinions? Wikis are like that, except they *actively encourage* random people -to interrupt the teacher with their own interjections. - -I can hear the objections now: "But putting our docs on a wiki means *anyone* -can fix typos!" - -Jesus. Christ. - -"It makes it easy to fix typos" is a horrible argument for using a wiki. - -First of all, as JKM says, you should have an editor (or at least someone to -proofread) which will catch a lot of the typos. - -And even if there *are* typos, they're one of the least important things you -need to worry about anyway. Misspelling "their" isn't going to impact the -effectiveness of your teaching very much. Your lessons being a disorganized -mess because they were written by three different people across six months *is* -going to make them less effective. - -Keeping your documentation in a wiki also makes it hard or impossible to keep it -where it belongs: in version control right alongside your code. - -But all that is irrelevant because aside from Wikipedia itself and video game -wikis, *they don't fucking work*. - -The project maintainer sets up a wiki, sits back and pats herself on the back -saying: "I have set up a way for other people to do this boring job of writing -documentation for me. Now we wait." - -Maybe one or two people fix some typos. A dude who thinks he understands -a topic but actually doesn't writes some completely wrong docs. Maybe they get -reverted, maybe they don't. - -The project changes. A new user reads some of the (sparse) documentation which -is now out of date. Eventually they discover this and complain only to be met -with: "Well it's a wiki, fix it yourself!" - -It is not the responsibility of the student to fix a broken lesson plan. For -fuck's sake, *the entire point of having a teacher* is that they know what the -students need to learn and the students don't! - -It's completely okay to ask your students for criticism so you can improve your -lesson plan. Asking "what parts did you find difficult?" is fine. It's another -thing entirely to ask them to *write your lesson plan for you*. - -Seriously: fuck wikis. They are bad and terrible. Do not use them. Take the -time and effort to write some real documentation instead. - -## The Reference - -The final type of documentation is "the reference". This section is for the -users who have traveled through the "hairball" and made it to the other side. -They're now your experts, and the reference should support them as they use your -project in their daily work. - -This section should contain things that experienced users are likely to need, -such as: - -* "API documentation" for every user-facing part of your project. -* A full changelog, with particular attention to backwards-incompatible changes - between versions. -* Details about the internal implementation of the project. -* Contribution policies (if your project accepts outside contributions). - -Tools like JavaDoc can produce something that looks like the first, but I share -the same opinion as Jacob Kaplan-Moss: - -> Auto-generated documentation is almost worthless. At best it's a slightly -> improved version of simply browsing through the source, but most of the time -> it's easier just to read the source than to navigate the bullshit that these -> autodoc tools produce. About the only thing auto-generated documentation is -> good for is filling printed pages when contracts dictate delivery of a certain -> number of pages of documentation. I feel a particularly deep form of rage -> every time I click on a "documentation" link and see auto-generated -> documentation. -> -> There's no substitute for documentation written, organized, and edited by -> hand. - -Yes, you can probably find a tool to read your project's source and shit out -some HTML files with the function names in them. Maybe it will even include the -docstrings! - -I would still urge you to write your API docs by hand. It's going to be -a little more typing, but the results will be much better for a number of -reasons. - -API docs and docstrings, while similar, serve different purposes. Docstrings -have to provide what you need in the heat of coding in a REPL-friendly format. -API docs can afford the luxury of a bit more explanation, as well as links to -other things the user might want to know while browsing them on their couch. -API docs should also be Google-friendly. - -A common objection here is that you're going to be retyping a lot of words. -Copy and paste mostly solves that problem, and learning to type makes the rest -a non-issue. - -Some will say: "But copy and pasting is evil! You're duplicating effort! How -will you keep the changes in the docstrings and the API docs in sync if they -change?" - -My opinion here is that if your public-facing API is changing often, you're -probably going to be making your users' lives harder when they need to -constantly update their code to work with yours. So the least you can do is -make *your* life a little harder to provide them with the best documentation -possible to help ease the pain. - -Auto-generated documentation has no coherent voice. It pulls in everything in -the code without regard for overall structure and vision. You can *probably* -get away with it for the API docs in your "reference" documentation, or you -could take some pride in your work and write the best docs possible! - -## Act 7: "A New Hope" - -The final act of our play is set in a mall parking lot on a Sunday afternoon. -A single car is in the parking lot. Inside is a family: a mother and father who -are teaching their son to drive. - -They start by driving the car into the middle of the lot, away from any -obstacles. The son gets into the driver's seat, and the parents explain briefly -what the main controls do. They let him drive around the empty lot a bit to get -a feel for how the car works. - -When it's time for him to park he shifts to park and takes off his seatbelt. -His mom reminds him of the control called a "parking brake". He realizes that -he should use this when parking. A set of neurons is now linked in his brain -and he will remember to use the parking brake properly for the rest of his life. - -Over time the parents take their son driving many times, always being sure that -they're putting him into situations he can handle (but will still learn from). -He drives on a road, then learns to parallel park, then drives on a highway. - -He has questions along the way. Sometimes the parents are ready with an answer. -Sometimes the questions reveal something else missing deeper down in his -knowledge which the parents correct. - -Over time he learns more and more. He gets his license and begins driving on -his own. - -When he gets a flat tire he reads the owner's manual and fixes it. - -He watches the How It's Made episode about his car because he's curious how the -brakes which saved his life at a stop sign last week actually work. - -His windshield wipers stop working one day. He opens up the hood and figures -out the problem, fixing it himself. - -One day he is hit by a drunk driver. He walks away with only bruises. He never -saw the countless crash tests the engineers performed to create the airbag -system, but they saved his life. - -In the last scene we see the son many years later. His hair is a bit gray now, -but otherwise he looks a lot like the teenager who forgot to use the parking -brake. - -He's in a car with his teenage daughter, and he's teaching her how to drive. - - {% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2013/09/teach-dont-tell.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2013/09/teach-dont-tell.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,742 @@ ++++ +title = "Teach, Don't Tell" +snip = "Documentation Matters" +date = 2013-09-03T10:55:00Z +draft = false + ++++ + +This post is about writing technical documentation. More specifically: it's +about writing documentation for programming languages and libraries. + +I love reading great documentation. When I have a question and the +documentations explains the answer almost as if the author has magically +anticipated my problem, I get a warm, fuzzy feeling inside. I feel a connection +with the writer that makes me smile. + +I also love writing documentation. Being able to rewire the neurons in +someone's brain so that they understand something they didn't understand before +is extremely satisfying. Seeing (or hearing about) the "click" when a bunch of +concepts suddenly fall together and make sense never fails to make my day. + +This post is going to be about what I think good documentation is and how +I think you should go about writing it. I'm not perfect, so you should take +everything with a grain of salt, but I hope you'll find it useful and +thought-provoking even if you don't agree with me about everything. + +I'd like to say thanks to [Craig Zheng][cz] and [Honza Pokorny][honza] for +proofreading this. + +[cz]: http://craigzheng.com/ +[honza]: http://honza.ca/ + +{{% toc %}} + +## Prior Reading + +Before you read this post there are two other things I think you should read +first. + +The first is Jacob Kaplan-Moss' [Writing Great Documentation][] series. He's +certainly more qualified than I am to write about this stuff, so you should +check that out if you haven't already. A lot of what I say here is going to +agree with and build on the ideas he talked about. + +The other thing you should read is [The Science of Scientific Writing][] by +George Gopen and Judith Swan. Don't be put off by the fact that it's written +for scientists publishing papers in journals. Everything in that article +applies equally well to programmers writing technical docs. Read the entire +thing. It's worth it. + +[Writing Great Documentation]: http://jacobian.org/writing/great-documentation/ +[The Science of Scientific Writing]: http://www.americanscientist.org/issues/id.877,y.0,no.,content.true,page.1,css.print/issue.aspx + +## Why Do We Document? + +Let's get started. The first thing to nail down is *why* we're documenting +a programming language or library in the first place. There are many things you +might want to accomplish, but I'm going to boil them down into a single +statement: + +**The purpose of technical documentation is to take someone who has never seen +your project, teach them to be an expert user of it, and support them once they +become an expert.** + +At first glance this probably doesn't seem too controversial or interesting. +But there's one word in there that makes *all* the difference, and it frames my +entire perspective on documentation. + +## Teaching + +If you want to take a person who has never played the guitar and turn them into +a virtuoso guitarist, how can you do that? + +You *teach* them. + +If you want to take a high school student and turn them into a computer +scientist, how can you do that? + +You *teach* them. + +If you want to take a programmer who has never seen your library before and turn +them into an expert user of it, how can you do that? + +You *teach* them! + +Guitar lessons are usually taught in person, one-on-one, with a teacher. +Computer Science is usually taught by professors in classrooms. Programming +library usage is usually taught by documentation. + +If the goal of documentation is to turn novices into experts, then *the +documentation must teach*. You should think of your documentation as a lesson +(or series of lessons) because *that's what it is*. + +When writing technical documentation you (usually) don't have the advantage of +having a one-on-one dialog with the learners. This makes it a bit more +difficult, but not impossible as long as you're careful. Your documentation +needs to fill the role of both the in-person lessons *and* the textbook. + +The rest of this post will be almost entirely about how to apply the +"documentation is teaching" mindset to writing programming docs. + +## A Play in Seven Acts + +I'm going to break up the content of this post with some venting about *bad* +documentation. If you want to skip these little rants, go ahead. + +Each act in our play has two characters: a teenager and a parent. The teenager +has just turned sixteen and would like to learn to drive so they can hang out +with their friends without relying on their parents to drive them everywhere. + +Each act will demonstrate a caricature of a particularly *bad* form of +documentation. I hope these little metaphors will help show why certain forms +of documentation are ineffective cop-outs and why you should write *real* +documentation instead. + +## Act 1: "Read the Source" + +Our play starts with a son and father sitting at the breakfast table. The son +is munching on some cereal before school while the father reads his iPad before +leaving for work. + +The son says: "Hey Dad, you said you were going to teach me how to drive after +school today. Are we still going to do that?" + +The father, without looking up from his iPad, replies: "Of course, son. The car +is in the garage and I laid out a set of wrenches on the workbench. Take the +car apart and look at each piece, then put it back together. Once you've done +that I'll take you to the DMV for your driving test." + +The son quietly continued eating his cereal. + +If you use many open source libraries you've undoubtedly encountered some whose +README says something like "read the source". Every time I see one, I die +a little bit inside. + +Source code is *not* documentation. Can you learn to be a guitarist by simply +listening to a piece of music intently? Can you become a painter by visiting +a lot of museums? Of course not! + +Let me be clear: I'm not trying to say that reading source code isn't a valuable +thing to do. It is! + +Looking at other artists' paintings is extremely useful *once you know how to +paint*. Learning how the brakes of a car are constructed can save your life, +*once you know how to drive*. + +Once your library's users know how to work with it, reading its source is +absolutely worth their time. But you can't look at the finished product and +understand the perspective and ideas that went into it without additional +guidance. That's a job for documentation. + +## Tools of the Trade + +Writing good documentation doesn't take much in the way of tools. + +For example: you don't need a thesaurus. Don't try to avoid using the same +words by substituting synonyms. Just talk to your users like you would talk to +another human being in real life! + +If you use the same word ten times in a row your readers probably won't even +notice. But I guarantee they're going to notice if you throw in strange, +uncommon words for no good reason. + +There are two tools I *will* recommend before moving on. The first isn't +actually a tool, but a skill. + +To write great documentation, you need to be able to type. + +When you write docs you'll inevitably write yourself into a corner and realize +you need to take a new direction. If you don't type quickly, you might be +hesitant to throw away writing that doesn't really work. You need to learn to +type well so you don't feel bad throwing away a chunk of a thousand words that +don't fit. + +Steve Yegge's article [Programming's Dirty Little Secret][type-dammit] is +a great rant on this topic. + +You should also get yourself a nice keyboard. A good keyboard won't make you +a good writer (just like a good guitar won't make you a good guitarist), but it +*will* make you want to write more just for the sheer joy of using +a well-designed piece of equipment. + +I started practicing guitar a lot more after I got a new guitar that was much +nicer than my old one. If you spend a hundred dollars, get a nice keyboard, and +end up wanting to write more, it was worth it! + +Be thankful that a nice keyboard only costs $100 to $300 and not several +thousand dollars like a nice instrument. + +[type-dammit]: http://steve-yegge.blogspot.com/2008/09/programmings-dirtiest-little-secret.html + +## Act 2: "Read the Tests" + +The next scene opens with a mother picking her daughter up from high school. + +"Hi Mom", she says, "are you still going to teach me to drive today?" + +"Yep!" she replies. "Let's get going." + +After ten minutes of driving they arrive at the Chevrolet factory. + +The girl looks around, puzzled. She asks: "What are we doing here?" + +The mother smiles and says: "You're in luck, honey, my friend Jim works here at +the Chevy plant, and he's gonna let you watch a few crash tests of the new +Malibu! Once you see a few cars smash into each other, I'll take you down to +the DMV for your driving test." + +Another common form of "documentation" is the README instructing users to "read +the tests". + +Tests aren't docs. + +Again, let's be clear: once you already know how to use a library, reading the +tests is *very* useful. But you need documentation to make you an expert user +first! + +You don't learn to drive by watching crash tests. But learning how your car +behaves during a crash can save your life *once you know how to drive*. + +A common argument I see goes something like this: + +"The tests use the library, so they're a good example of how to use it!" + +This is true in some very superficial sense, but completely misses the mark. + +Most of the tests are probably going to deal with edge cases. Edge cases are +things a normal user won't be encountering very often (otherwise they wouldn't +be edge cases!). + +If you're lucky, you might get a test that verifies the library works correctly +on a normal set of input. But a "normal set of input" is what the users are +going to be working with the majority the time! + +Tests simply aren't a good guide to what a user is going to be encountering on +a day-to-day basis. They can't teach a novice to be an expert. + +## How to Teach + +If you accept my idea that the purpose of documentation is to *teach* users, the +next question is obviously: "How do I teach my users?" + +I've been lucky enough to have the chance to teach dancing semi-formally for +around 6 or 7 years, and lots of various other things informally for a long +time. The only way to *really* learn how to teach is to *do it*. + +There's no substitute for sitting down with someone face-to-face and teaching +them something. **If you want to write better documentation, you need to +practice teaching**. + +I'm not talking about writing out lesson plans or anything nearly so formal. Do +you have a hobby (not programming)? If so, spend a couple of hours on a weekend +teaching a friend about it. You'll get some practice teaching and they'll get +to learn something new. + +(If you don't have any non-programming hobbies, maybe you should find some.) + +If you like photography, teach someone the basics of exposure and composition. +If you dance, teach them some basic steps. If you play an instrument, teach +them how to play a simple song. If you like camping, teach them what all the +gear is for. You get the idea. + +Don't go overboard. You don't need to give someone a degree, you just need to +practice teaching a little bit. You need to practice the art of rewiring +someone's neurons with your words. + +Once you jump into teaching something (even something simple) you'll probably +realize that although you know how to do it yourself, it's a lot harder to teach +someone else. + +This is obvious when you're working face-to-face with someone. When you tell +them how to play a C major chord on the guitar and they only produce a strangled +squeak, it's clear that you need to slow down and talk about how to press down +on the strings properly. + +As programmers, we almost *never* get this kind of feedback about our +documentation. We don't see that the person on the other end of the wire is +hopelessly confused and blundering around because they're missing something we +thought was obvious (but wasn't). Teaching someone in person helps you learn to +anticipate this, which will pay off (for your users) when you're writing +documentation. + +With all that said, I do want to also talk a little about the actual process of +teaching. + +The best description of how to teach that I've seen so far is from the book [How +to Solve It][]. Everyone who wants to teach should read this book. The passage +that really jumped out at me is right in the first page of the first chapter: + +> The best [way for the teacher to help their student] is to help the student +> naturally. The teacher should put himself in the student's place, he should +> see the student's case, he should try to understand what is going on in the +> student's mind, and ask a question or indicate a step that *could have +> occurred to the student himself*. + +This, right here, is the core of teaching. This is it. This is how you do it. + +People don't learn by simply absorbing lots of unstructured information as it's +thrown at them. You can't read a Spanish dictionary to someone to teach them +Spanish. + +When you want to teach someone you need to put yourself in their shoes and walk +along the path with them. Hold their hand, guide them around the dangerous +obstacles and catch them when they fall. *Don't* carry them. *Certainly +don't* just drive them to the destination in your car! + +The process needs to go something like this: + +1. Figure out what they already know. +2. Figure out what you want them to know after you finish. +3. Figure out a single idea or concept that will move state 1 a little bit + closer to state 2. +4. Nudge the student in the direction of that idea. +5. Repeat until state 1 becomes state 2. + +Too often I see documentation that has very carefully considered step 2, and +then simply presents it to the reader as a pronouncement from God. That isn't +teaching. That's telling. People don't learn by being *told*, they +learn by being *taught*. + +[How to Solve It]: http://www.amazon.com/dp/069111966X/?tag=stelos-20 + +## Act 3: "Literate Programming" + +The third act opens with a daughter talking to her mother the day before her +sixteenth birthday. + +"Hey Mom," she says, "I don't know if you got me a present yet, but if not, what +I'd *really* like for my birthday are driving lessons." + +The mother smiles and says: "Don't worry, it's all taken care of. Just wait for +tomorrow." + +The next day at her birthday party she unwraps the present from her mom. Inside +is a DVD of the show How It's Made. She looks quizzically at her mother. + +"That DVD has an episode about the factory that builds your car! Once you watch +the whole thing I'll take you for your driving test." + +A horrible trend I've noticed lately is using "literate programming" tools like +Docco, Rocco, etc and telling users to read the results for documentation. + +Programming languages and libraries are tools. Knowing how a tool was made +doesn't mean you know how to use it. When you take guitar lessons, you don't +visit a luthier to watch her shape a Telecaster out of Ash wood. + +Knowing how your car was built can help you, *once you know how to drive*. + +Knowing how your guitar was built can help you, *once you know how to play*. + +A common theme throughout these acts/rants is that all of these things I'm +picking on (source, tests, literate programming, and more) are good things with +real benefits *once you have actual documentation in place to teach users*. + +But until that happens, they're actually *bad* because they let you pretend +you've written documentation and your job is done (JKM mentions this in his +series). Your job is not done until you've taught your users enough to become +experts. *Then* they can take advantage of all these extras. + +## The Anatomy of Good Documentation + +The rest of this post is going to be about the individual components that make +up good documentation. My views are pretty similar to JKM's, so if you haven't +read the series I mentioned in the first section you should probably do that. + +In my mind I divide good documentation into roughly four parts: + +1. First Contact +2. The Black Triangle +3. The Hairball +4. The Reference + +There don't necessarily have to be four separate documents for each of these. +In fact the first two can usually be combined into a single file, while the last +two should probably be split into many pieces. But I think each component is +a distinct, important part of good documentation. + +Let's take a look at each. + +## First Contact + +When you release a new programming language or library into the wild, the +initial state of your "users" is going to be blank. The things they need +to know when they encounter your library are: + +1. What is this thing? +2. Why would I care about this thing? +3. Is it worth the effort to learn this thing? + +Your "first contact" documentation should explain these things to them. + +You don't need to explain things from first principles. Try to put yourself in +the shoes of your users. When you're teaching your teenager to drive, you don't +need to explain what a "wheel" is. They probably have some experience with +"things on wheels that you move around in" like lawn mowers or golf carts (or +even video games). + +Likewise: if you're creating a web framework, most of the people that stumble on +to your project are probably going to know what "HTML" is. It's good to err +a little bit on the side of caution and explain a little more than to assume too +much, but you can be practical here. + +Your "first contact" docs should explain what, in plain words, your thing does. +It should show someone why they should care about that. Will it save them time? +Will it take more time, but be more stable in exchange? Is it just plain fun? + +For bonus points, you can also mention why someone might want to *not* use your +project. Barely anyone ever mentions the tradeoffs involved with using their +work, so to see a project do this is refreshing. + +Finally, the user needs to know if it's worth spending some of their finite +amount of time on this planet learning more about your project. You should +explicitly spell out things like: + +* What license the project uses (so they know if it's practical to use). +* Where the bug tracker is (so they can see issues). +* Where the source code is (so they can see if it's (relatively) recently + maintained). +* Where the documentation is (so they can skim it and get an idea of the effort + that's going to be involved in becoming an expert). + + + +## Act 4: "Read the Docstrings" + +Scene four. A father is finally making good on his promise to give his daughter +driving lessons. + +"Okay Dad," she says, "I'm ready. I've never driven a car before. Where do we +start?" + +A woman in her mid-forties walks through the door. "Who's this?" the daughter +asks. + +"This is your driving teacher, Ms. Smith." the father replies. "She's going to +sit in the passenger seat with you while you drive the two hour trip to visit +grandpa. If you have any questions about a part of the car while you're +driving, you can ask her and she'll tell you all about that piece. Here are +the keys, good luck!" + +In languages with [docstrings][] there's a tendency to write great docstrings +and call them documentation. I'm sure the "doc" in the word "docstrings" +contributes to this. + +Docstrings don't provide any organization or order (beyond "the namespace they +happen to be implemented in"). Users need to somehow know the name of the +function they need to even be able to *see* the docstring, and they can't know +that unless you *teach* them. + +Again, docstrings are great *once you know the project*. But when you're +teaching a novice how to use your library, you need to guide them along they +way and not sit back and answer questions when they manage to guess a magic +word correctly. + +[docstrings]: https://en.wikipedia.org/wiki/Docstring + +## The Black Triangle + +The next important piece of documentation is [the "black triangle"][]. It +should be a relatively short guide to getting your project up and running so the +user can poke at it. + +This serves a couple of purposes. First, it lets the user verify that yes, this +collection of bytes is actually going to run and *do something* on their +machine. It's a quick sanity check that the project hasn't bit rotted and is +still viable to use at that point in time. More importantly, it lets your +prospective user [get some paint on the canvas][paint]. + +Imagine if you went to your first guitar lesson and the teacher said: "Okay, +we're going to start by learning 150 different chords. Then in about six months +we can play some songs." No guitar teacher does that. They teach you three +chords and give you a couple of cheesy pop songs to play. It helps the student +get a feel for what being a guitarist as a whole is going to be like, and it +gives them something to help keep their interest. + +Your "black triangle" documentation should be a short guide that runs the user +through the process of retrieving, installing, and poking your project or +language. + +"Short" here is a relative word. Some projects are going to require more setup +to get running. If the benefits are enough to justify the effort, that's not +necessarily a problem. But try to keep this as short as possible. *Just get +something on the screen* and move on. + +[the "black triangle"]: http://rampantgames.com/blog/2004/10/black-triangle.html +[paint]: http://worrydream.com/LearnableProgramming/#react + +## Act 5: "Read the API Docs" + +Our next scene opens a year after the last, with the father from the last scene +talking to his son. + +(Sadly, the daughter in that scene died in a car crash because she didn't know +to ask Ms. Smith about seatbelts before getting on the expressway. Ms. Smith +was wearing hers, of course.) + +"Okay son, I know you're a little scared of driving because of what happened to +your sister, but I've fixed the problem." + +He hands the young man an inch-thick book. "Asking Ms. Smith questions along +the way clearly didn't work, so we had her write out a paragraph or two about +each piece of your car. Go ahead and read the entire manual cover to cover and +then drive down to see grandpa." + +API documentation is like the user's manual of a car. When something goes wrong +and you need to replace a tire it's a godsend. But if you're learning to drive +it's not going to help you because *people don't learn by reading alphabetized +lists of disconnected information*. + +If you actually try to teach someone to use your project face-to-face, you'll +probably find yourself talking about things in one namespace for a while, then +switching to another to cover something related, then switching back to the +first. Learning isn't a straight path through the alphabet, it's a zig-zaggy +ramble through someone else's brain. + +## The Hairball + +This brings me to the next type of documentation: "the hairball". By now the +user has hopefully seen the "first contact" docs and the "black triangle" docs. +You've got them hooked and ready to learn, but they're still novices. + +The "hairball" is the twisted, tangled maze of teaching that is going to take +these novices and turn them into expert users. It's going to mold their brains, +one nudge at a time, until they have a pretty good understanding of how your +project works. + +You'll usually want to organize the "hairball" into sections (unless this is +a very small project). These sections will probably *kind of* line up with +namespaces in your project's public API, but when it makes sense to deviate you +should do so. + +Don't be afraid to write. Be concise but err on the side of explaining a bit +too much. Programmers are pretty good at skimming over things they already +know, but if you forget to include a crucial connection it can leave your users +lost and stumbling around in the woods. + +You should have a table of contents that lists each section of the "hairball". +And then each section should have its own table of contents that lists the +sections inside it. A table of contents is a wonderful way to get a birds-eye +view of what you're about to read, to prepare your brain for it. And of +course it's also handy for navigating around. + +This is where your hobby-teaching practice and your reading of How to Solve It +are going to come in handy. Put yourself in a user's brain and figure out each +little connective leap they're going to need to make to become an expert. + +## Act 6: "Read the Wiki" + +In the penultimate scene, a mother has signed her teenage son up for an +after-school driving class. + +On the first day, the teacher hands them a syllabus detailing what they're going +to cover, talks about grading, and sends them home a bit early. + +On the second day, she gives them a brief overview of the various pieces of +a car and how they work together. She also talks about a few of the most +important laws they'll need to be aware of. + +On the third day, the teacher calls in sick and they have a substitute. He +covers the material for half of the fifth day in the syllabus. He has to leave +early, so he brings in his nineteen year old daughter to finish the class. +She covers the first half of the fourth day's material. + +The fourth day the students arrive to find a note on the door saying the class +has been cancelled because the teacher is still sick and they can't find +a substitute. There's a note saying "TODO: we'll talk about the material +later." + +The fifth day the teacher has partially recovered, so she returns and covers the +material for the fifth day. It's a bit hard to understand her because she's had +half a bottle of Nyquil and is slurring most of her words and keeps saying +"cat" instead of "car". + +All the students fail the driving test. + +Wikis are an abomination. They are the worst form of "documentation". + +First of all: assuming they work as intended, they have no coherent voice. + +Have you ever taken a class with multiple teachers at once? Probably not, +because it doesn't work very well (with exceptions for things like partnered +dancing where there are distinct lead/follow parts). + +Worse still: have you ever taken a class where there's one jackass in the room +who keeps constantly raising his hand and offering his own (often incorrect) +opinions? Wikis are like that, except they *actively encourage* random people +to interrupt the teacher with their own interjections. + +I can hear the objections now: "But putting our docs on a wiki means *anyone* +can fix typos!" + +Jesus. Christ. + +"It makes it easy to fix typos" is a horrible argument for using a wiki. + +First of all, as JKM says, you should have an editor (or at least someone to +proofread) which will catch a lot of the typos. + +And even if there *are* typos, they're one of the least important things you +need to worry about anyway. Misspelling "their" isn't going to impact the +effectiveness of your teaching very much. Your lessons being a disorganized +mess because they were written by three different people across six months *is* +going to make them less effective. + +Keeping your documentation in a wiki also makes it hard or impossible to keep it +where it belongs: in version control right alongside your code. + +But all that is irrelevant because aside from Wikipedia itself and video game +wikis, *they don't fucking work*. + +The project maintainer sets up a wiki, sits back and pats herself on the back +saying: "I have set up a way for other people to do this boring job of writing +documentation for me. Now we wait." + +Maybe one or two people fix some typos. A dude who thinks he understands +a topic but actually doesn't writes some completely wrong docs. Maybe they get +reverted, maybe they don't. + +The project changes. A new user reads some of the (sparse) documentation which +is now out of date. Eventually they discover this and complain only to be met +with: "Well it's a wiki, fix it yourself!" + +It is not the responsibility of the student to fix a broken lesson plan. For +fuck's sake, *the entire point of having a teacher* is that they know what the +students need to learn and the students don't! + +It's completely okay to ask your students for criticism so you can improve your +lesson plan. Asking "what parts did you find difficult?" is fine. It's another +thing entirely to ask them to *write your lesson plan for you*. + +Seriously: fuck wikis. They are bad and terrible. Do not use them. Take the +time and effort to write some real documentation instead. + +## The Reference + +The final type of documentation is "the reference". This section is for the +users who have traveled through the "hairball" and made it to the other side. +They're now your experts, and the reference should support them as they use your +project in their daily work. + +This section should contain things that experienced users are likely to need, +such as: + +* "API documentation" for every user-facing part of your project. +* A full changelog, with particular attention to backwards-incompatible changes + between versions. +* Details about the internal implementation of the project. +* Contribution policies (if your project accepts outside contributions). + +Tools like JavaDoc can produce something that looks like the first, but I share +the same opinion as Jacob Kaplan-Moss: + +> Auto-generated documentation is almost worthless. At best it's a slightly +> improved version of simply browsing through the source, but most of the time +> it's easier just to read the source than to navigate the bullshit that these +> autodoc tools produce. About the only thing auto-generated documentation is +> good for is filling printed pages when contracts dictate delivery of a certain +> number of pages of documentation. I feel a particularly deep form of rage +> every time I click on a "documentation" link and see auto-generated +> documentation. +> +> There's no substitute for documentation written, organized, and edited by +> hand. + +Yes, you can probably find a tool to read your project's source and shit out +some HTML files with the function names in them. Maybe it will even include the +docstrings! + +I would still urge you to write your API docs by hand. It's going to be +a little more typing, but the results will be much better for a number of +reasons. + +API docs and docstrings, while similar, serve different purposes. Docstrings +have to provide what you need in the heat of coding in a REPL-friendly format. +API docs can afford the luxury of a bit more explanation, as well as links to +other things the user might want to know while browsing them on their couch. +API docs should also be Google-friendly. + +A common objection here is that you're going to be retyping a lot of words. +Copy and paste mostly solves that problem, and learning to type makes the rest +a non-issue. + +Some will say: "But copy and pasting is evil! You're duplicating effort! How +will you keep the changes in the docstrings and the API docs in sync if they +change?" + +My opinion here is that if your public-facing API is changing often, you're +probably going to be making your users' lives harder when they need to +constantly update their code to work with yours. So the least you can do is +make *your* life a little harder to provide them with the best documentation +possible to help ease the pain. + +Auto-generated documentation has no coherent voice. It pulls in everything in +the code without regard for overall structure and vision. You can *probably* +get away with it for the API docs in your "reference" documentation, or you +could take some pride in your work and write the best docs possible! + +## Act 7: "A New Hope" + +The final act of our play is set in a mall parking lot on a Sunday afternoon. +A single car is in the parking lot. Inside is a family: a mother and father who +are teaching their son to drive. + +They start by driving the car into the middle of the lot, away from any +obstacles. The son gets into the driver's seat, and the parents explain briefly +what the main controls do. They let him drive around the empty lot a bit to get +a feel for how the car works. + +When it's time for him to park he shifts to park and takes off his seatbelt. +His mom reminds him of the control called a "parking brake". He realizes that +he should use this when parking. A set of neurons is now linked in his brain +and he will remember to use the parking brake properly for the rest of his life. + +Over time the parents take their son driving many times, always being sure that +they're putting him into situations he can handle (but will still learn from). +He drives on a road, then learns to parallel park, then drives on a highway. + +He has questions along the way. Sometimes the parents are ready with an answer. +Sometimes the questions reveal something else missing deeper down in his +knowledge which the parents correct. + +Over time he learns more and more. He gets his license and begins driving on +his own. + +When he gets a flat tire he reads the owner's manual and fixes it. + +He watches the How It's Made episode about his car because he's curious how the +brakes which saved his life at a stop sign last week actually work. + +His windshield wipers stop working one day. He opens up the hood and figures +out the problem, fixing it himself. + +One day he is hit by a drunk driver. He walks away with only bruises. He never +saw the countless crash tests the engineers performed to create the airbag +system, but they saved his life. + +In the last scene we see the son many years later. His hair is a bit gray now, +but otherwise he looks a lot like the teenager who forgot to use the parking +brake. + +He's in a car with his teenage daughter, and he's teaching her how to drive. + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2015/07/nat-geo-a2540.html --- a/content/blog/2015/07/nat-geo-a2540.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,318 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "Bag Review: National Geographic A2540" - snip: "A great little shoulder bag for holding a DSLR kit." - created: 2015-07-24 18:42:00 - %} - - {% block article %} - -I posted this review on Imgur and Reddit a few days ago, but figured I would -blow the dust off my blog and post it here too. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-01.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-01.jpg) - -I got this bag a few months ago. I've used it to shoot around town and -flown/traveled with it a couple times. At first I wasn't thrilled with it, but -I've warmed up to it now and really like it. - -I bought it at [the Geographic Bags site][natgeo] for about $50 new. It's -sometimes [on Amazon][amazon] if Nat Geo is out of stock (affiliate link). - -I use it for two main things: a day bag for walking around town, and a "personal -item" on a plane where I load it down with all my camera gear to save the weight -in my carry on. - -[natgeo]: http://www.geographicbags.us/midi-satchel-for-personal-gear -[amazon]: http://www.amazon.com/dp/B003WE9MGO/?tag=stelos-20 - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-02.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-02.jpg) - -I like the branding/styling a lot. I'm pretty sure Nat Geo licenses the -branding to Manfrotto to actually make the bags, because when mine came the -return address on the shipping label was a Manfrotto warehouse. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-03.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-03.jpg) - -The construction is really nice. The stitching seems pretty solid, if a bit -uneven in places, but this is meant to be a working bag, not a piece of art. -The leather bits seem hearty. The buckles are okay (not amazing, but good -enough for the price). - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-04.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-04.jpg) - -Attachment points for the strap are solid. The strap is not removable, -unfortunately, but at the price I'd rather have a solid, permanent strap than -a removable one with flimsy connections. - -The strap itself is fine. It's strong and I don't see it breaking any time -soon. It's a little bit thin, so if you load the bag down a lot it can start to -dig into your shoulder a bit. Luckily... - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-05.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-05.jpg) - -There's an optional shoulder pad you can buy for the strap. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-06.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-06.jpg) - -It's got nice thick padding, and velcros around the strap. - -(Please excuse the cat hair in all the photos). - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-07.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-07.jpg) - -I use it when I've got more than a couple of pounds in the bag to save my -shoulders. When I'm using it as a day pack I take it off. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-08.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-08.jpg) - -The back handle is great for carrying it if you're wearing a backpack, yanking -it out of places, etc. - -There's also a little strap on the back -- it's designed for sliding the handle -of some rolling luggage through it so it can rest securely on top. The -stitching on the velcro bit on it is fraying a bit for me. I don't really care -because I never use it anyway. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-09.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-09.jpg) - -There are two small, sleek pockets on the front. They're not huge, but they'll -hold something wallet or small notebook-sized. - -I wouldn't keep a wallet in them though, because they're too easy to -access/pickpocket. - -For day trips I generally use them to hold lens caps while shooting, for lens -cleaning cloths, etc. When I fly I use them to hold a polarizer and neutral -density filters (77mm filters, in their plastic cases, will fit nicely (not -pictured, sorry, I forgot)). - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-10.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-10.jpg) - -Once you unbuckle the top flap there's still a zipper protecting the contents. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-11.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-11.jpg) - -The flap has some nicely patterned fabric to break up the brown. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-12.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-12.jpg) - -The front pouch is good for holding some small stuff. It's got dividers for -holding really thin things like pens and notebooks. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-13.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-13.jpg) - -The inside lining is nice and bright so it's easy to find small things. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-14.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-14.jpg) - -The bag itself has one big main compartment. There's one small padded divider -flap with a velcro bit to hold it down. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-15.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-15.jpg) - -It doesn't hold very much, but it's enough for a Kindle or small book. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-16.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-16.jpg) - -The bag also comes with a padded insert included. It's got the same pattern as -the other bits of the bag. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-17.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-17.jpg) - -There's a thin handle for yanking it out of the bag if you need to. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-18.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-18.jpg) - -Same golden lining. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-19.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-19.jpg) - -There's a padded divider inside that divides the insert into roughly 1/3 and 2/3 -sections. It's got a fold sewn in so you can fold it over a lens to protect the -top if necessary. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-20.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-20.jpg) - -That divider can be removed if you want to just use the whole space of the -insert. I never do. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-21.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-21.jpg) - -Nestled inside the main bag. It looks pretty snug... - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-22.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-22.jpg) - -But it doesn't take up quite all of the bag -- there's still room next to it for -something. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-23.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-23.jpg) - -That's it for the bag itself. Let's pack it! - -I like using this bag as my "personal item" on a plane. It's small enough to -fit under the seat, and if I put all my camera gear in it I save a ton of -weight/space in my carry on (I never check bags). - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-24.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-24.jpg) - -I shoot with a Pentax K5 II with a battery grip and L plate. Unfortunately this -bag isn't large enough to hold a gripped, L-plated DSLR with a lens attached -ready to go, so you've got to store the body separately. - -In practice this isn't a huge deal. I can take out the camera and carry it with -its strap as I shoot, and use the bag to hold the spare lenses. Then -I disconnect them once I'm finished shooting. It's a bit more work, but worth -it for the space savings. - -If you don't have a battery grip, the camera CAN be stored with a lens attached -in the camera insert. - -If you want to keep a gripped & L-plated DSLR with a lens attached in a shoulder -bag, the Think Tank Retro 10 is a beefy bag that will work. It's a lot bulkier -though, so it's best suited for non-flying. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-25.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-25.jpg) - -Three lenses that cover everything I need right now. 16-50mm, 50-135mm, and -200mm, all constant f/2.8 and weather sealed like the body. - -Sometimes I don't bother with the 200mm, which frees up a lot more space. Or -sometimes I take a few primes, which also frees up space. I just wanted to show -the max you could cram into the bag for this review. - -I'll probably ditch the 50-135 and 200 once the 70-200mm comes out for more -savings. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-26.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-26.jpg) - -The two big lenses go in the insert, the smaller one goes outside. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-27.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-27.jpg) - -Close the insert lid. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-28.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-28.jpg) - -I like to have just a bit of extra padding, so I grab an extra divider from -another bag. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-29.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-29.jpg) - -It lays on top of the end cap of the small zoom, just for some extra cushioning. -I'm probably just paranoid. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-30.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-30.jpg) - -I can't fly without headphones or I go insane. I like keeping them in my -personal item because it's easier to get them out once you're on the flight -without fucking around with the overhead bin. - -I use Marshall Monitors. I know they're big and bulky, but I can't stand using -earbuds for more than an hour or so at a time. These fold up nicely and have -a couple of other features that are really nice, so for me they're worth the -space. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-31.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-31.jpg) - -I nestle them on top of the small zoom. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-32.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-32.jpg) - -The body goes on top of the camera insert and big lenses. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-33.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-33.jpg) - -I tuck the strap (Black Rapid Metro) between the body and headphones. I could -store it separately, but I have the carabiner loc-tited shut so it'll never, -ever come off. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-34.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-34.jpg) - -Next necessity for flying: Kindle! - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-35.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-35.jpg) - -I could save a bit of space by ditching the case, but I like it... - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-36.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-36.jpg) - -It fits well behind the divider in the main section. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-37.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-37.jpg) - -Main section packed! - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-38.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-38.jpg) - -A few other miscellaneous things. - -Sometimes I charge both batteries the night before and ditch the charger. It -depends on how long I'm traveling for. - -I like to have my sunglasses in my personal item to use as a makeshift sleep -mask. Not quite a good as a dedicated one, but since I'm bringing them -anyway... - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-39.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-39.jpg) - -This is a pretty tight fit, but it does fit. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-40.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-40.jpg) - -Front packed! - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-41.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-41.jpg) - -The straps buckle (barely). - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-42.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-42.jpg) - -The main fabric of the bag seems like it wouldn't be too bad in the rain, but -I bought a rain cover for a National Geographic backpack, so I figured I'd try -it on this bag too. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-43.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-43.jpg) - -It fits. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-44.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-44.jpg) - -It's a bit loose because it's designed for a backpack, but you can cinch it up -with the elastic cord and it does the job. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-45.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-45.jpg) - -It'll fit in the front pocket too. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-46.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-46.jpg) - -Time for the final verdict. United's site says a personal item should fit -within 9" x 10" x 17". - -We're good on the short dimension. - -(Cardboard boxes to prove I'm not cheating.) - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-47.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-47.jpg) - -About half an inch past 10". I doubt I'll ever get called out on that -(especially since we have room to spare in the other dimensions), but it'll -squish a bit if absolutely necessary. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-48.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-48.jpg) - -Plenty of room to spare on the wide dimension. - -[![Photo](/media/images{{ parent_url }}/nat-geo-a2540-49.jpg)](/media/images{{ parent_url }}/full/nat-geo-a2540-49.jpg) - -That's it! Obviously you can mix and match stuff as needed (swap a lens for -a flash or water bottle, etc). - -I hope this was helpful! I'm quite happy with this little bag. It's great for -just walking around town, or turning my personal item into a brick of camera -gear to free up room in my carry on. - - - {% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2015/07/nat-geo-a2540.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2015/07/nat-geo-a2540.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,315 @@ ++++ +title = "Bag Review: National Geographic A2540" +snip = "A great little shoulder bag for holding a DSLR kit." +date = 2015-07-24T18:42:00Z +draft = false + ++++ + +I posted this review on Imgur and Reddit a few days ago, but figured I would +blow the dust off my blog and post it here too. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-01.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-01.jpg) + +I got this bag a few months ago. I've used it to shoot around town and +flown/traveled with it a couple times. At first I wasn't thrilled with it, but +I've warmed up to it now and really like it. + +I bought it at [the Geographic Bags site][natgeo] for about $50 new. It's +sometimes [on Amazon][amazon] if Nat Geo is out of stock (affiliate link). + +I use it for two main things: a day bag for walking around town, and a "personal +item" on a plane where I load it down with all my camera gear to save the weight +in my carry on. + +[natgeo]: http://www.geographicbags.us/midi-satchel-for-personal-gear +[amazon]: http://www.amazon.com/dp/B003WE9MGO/?tag=stelos-20 + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-02.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-02.jpg) + +I like the branding/styling a lot. I'm pretty sure Nat Geo licenses the +branding to Manfrotto to actually make the bags, because when mine came the +return address on the shipping label was a Manfrotto warehouse. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-03.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-03.jpg) + +The construction is really nice. The stitching seems pretty solid, if a bit +uneven in places, but this is meant to be a working bag, not a piece of art. +The leather bits seem hearty. The buckles are okay (not amazing, but good +enough for the price). + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-04.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-04.jpg) + +Attachment points for the strap are solid. The strap is not removable, +unfortunately, but at the price I'd rather have a solid, permanent strap than +a removable one with flimsy connections. + +The strap itself is fine. It's strong and I don't see it breaking any time +soon. It's a little bit thin, so if you load the bag down a lot it can start to +dig into your shoulder a bit. Luckily... + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-05.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-05.jpg) + +There's an optional shoulder pad you can buy for the strap. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-06.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-06.jpg) + +It's got nice thick padding, and velcros around the strap. + +(Please excuse the cat hair in all the photos). + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-07.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-07.jpg) + +I use it when I've got more than a couple of pounds in the bag to save my +shoulders. When I'm using it as a day pack I take it off. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-08.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-08.jpg) + +The back handle is great for carrying it if you're wearing a backpack, yanking +it out of places, etc. + +There's also a little strap on the back — it's designed for sliding the handle +of some rolling luggage through it so it can rest securely on top. The +stitching on the velcro bit on it is fraying a bit for me. I don't really care +because I never use it anyway. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-09.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-09.jpg) + +There are two small, sleek pockets on the front. They're not huge, but they'll +hold something wallet or small notebook-sized. + +I wouldn't keep a wallet in them though, because they're too easy to +access/pickpocket. + +For day trips I generally use them to hold lens caps while shooting, for lens +cleaning cloths, etc. When I fly I use them to hold a polarizer and neutral +density filters (77mm filters, in their plastic cases, will fit nicely (not +pictured, sorry, I forgot)). + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-10.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-10.jpg) + +Once you unbuckle the top flap there's still a zipper protecting the contents. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-11.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-11.jpg) + +The flap has some nicely patterned fabric to break up the brown. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-12.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-12.jpg) + +The front pouch is good for holding some small stuff. It's got dividers for +holding really thin things like pens and notebooks. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-13.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-13.jpg) + +The inside lining is nice and bright so it's easy to find small things. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-14.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-14.jpg) + +The bag itself has one big main compartment. There's one small padded divider +flap with a velcro bit to hold it down. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-15.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-15.jpg) + +It doesn't hold very much, but it's enough for a Kindle or small book. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-16.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-16.jpg) + +The bag also comes with a padded insert included. It's got the same pattern as +the other bits of the bag. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-17.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-17.jpg) + +There's a thin handle for yanking it out of the bag if you need to. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-18.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-18.jpg) + +Same golden lining. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-19.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-19.jpg) + +There's a padded divider inside that divides the insert into roughly 1/3 and 2/3 +sections. It's got a fold sewn in so you can fold it over a lens to protect the +top if necessary. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-20.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-20.jpg) + +That divider can be removed if you want to just use the whole space of the +insert. I never do. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-21.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-21.jpg) + +Nestled inside the main bag. It looks pretty snug... + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-22.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-22.jpg) + +But it doesn't take up quite all of the bag — there's still room next to it for +something. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-23.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-23.jpg) + +That's it for the bag itself. Let's pack it! + +I like using this bag as my "personal item" on a plane. It's small enough to +fit under the seat, and if I put all my camera gear in it I save a ton of +weight/space in my carry on (I never check bags). + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-24.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-24.jpg) + +I shoot with a Pentax K5 II with a battery grip and L plate. Unfortunately this +bag isn't large enough to hold a gripped, L-plated DSLR with a lens attached +ready to go, so you've got to store the body separately. + +In practice this isn't a huge deal. I can take out the camera and carry it with +its strap as I shoot, and use the bag to hold the spare lenses. Then +I disconnect them once I'm finished shooting. It's a bit more work, but worth +it for the space savings. + +If you don't have a battery grip, the camera CAN be stored with a lens attached +in the camera insert. + +If you want to keep a gripped & L-plated DSLR with a lens attached in a shoulder +bag, the Think Tank Retro 10 is a beefy bag that will work. It's a lot bulkier +though, so it's best suited for non-flying. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-25.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-25.jpg) + +Three lenses that cover everything I need right now. 16-50mm, 50-135mm, and +200mm, all constant f/2.8 and weather sealed like the body. + +Sometimes I don't bother with the 200mm, which frees up a lot more space. Or +sometimes I take a few primes, which also frees up space. I just wanted to show +the max you could cram into the bag for this review. + +I'll probably ditch the 50-135 and 200 once the 70-200mm comes out for more +savings. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-26.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-26.jpg) + +The two big lenses go in the insert, the smaller one goes outside. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-27.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-27.jpg) + +Close the insert lid. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-28.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-28.jpg) + +I like to have just a bit of extra padding, so I grab an extra divider from +another bag. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-29.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-29.jpg) + +It lays on top of the end cap of the small zoom, just for some extra cushioning. +I'm probably just paranoid. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-30.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-30.jpg) + +I can't fly without headphones or I go insane. I like keeping them in my +personal item because it's easier to get them out once you're on the flight +without fucking around with the overhead bin. + +I use Marshall Monitors. I know they're big and bulky, but I can't stand using +earbuds for more than an hour or so at a time. These fold up nicely and have +a couple of other features that are really nice, so for me they're worth the +space. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-31.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-31.jpg) + +I nestle them on top of the small zoom. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-32.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-32.jpg) + +The body goes on top of the camera insert and big lenses. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-33.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-33.jpg) + +I tuck the strap (Black Rapid Metro) between the body and headphones. I could +store it separately, but I have the carabiner loc-tited shut so it'll never, +ever come off. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-34.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-34.jpg) + +Next necessity for flying: Kindle! + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-35.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-35.jpg) + +I could save a bit of space by ditching the case, but I like it... + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-36.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-36.jpg) + +It fits well behind the divider in the main section. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-37.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-37.jpg) + +Main section packed! + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-38.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-38.jpg) + +A few other miscellaneous things. + +Sometimes I charge both batteries the night before and ditch the charger. It +depends on how long I'm traveling for. + +I like to have my sunglasses in my personal item to use as a makeshift sleep +mask. Not quite a good as a dedicated one, but since I'm bringing them +anyway... + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-39.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-39.jpg) + +This is a pretty tight fit, but it does fit. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-40.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-40.jpg) + +Front packed! + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-41.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-41.jpg) + +The straps buckle (barely). + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-42.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-42.jpg) + +The main fabric of the bag seems like it wouldn't be too bad in the rain, but +I bought a rain cover for a National Geographic backpack, so I figured I'd try +it on this bag too. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-43.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-43.jpg) + +It fits. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-44.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-44.jpg) + +It's a bit loose because it's designed for a backpack, but you can cinch it up +with the elastic cord and it does the job. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-45.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-45.jpg) + +It'll fit in the front pocket too. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-46.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-46.jpg) + +Time for the final verdict. United's site says a personal item should fit +within 9" x 10" x 17". + +We're good on the short dimension. + +(Cardboard boxes to prove I'm not cheating.) + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-47.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-47.jpg) + +About half an inch past 10". I doubt I'll ever get called out on that +(especially since we have room to spare in the other dimensions), but it'll +squish a bit if absolutely necessary. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-48.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-48.jpg) + +Plenty of room to spare on the wide dimension. + +[![Photo](/media/images/blog/2015/07/nat-geo-a2540-49.jpg)](/media/images/blog/2015/07/full/nat-geo-a2540-49.jpg) + +That's it! Obviously you can mix and match stuff as needed (swap a lens for +a flash or water bottle, etc). + +I hope this was helpful! I'm quite happy with this little bag. It's great for +just walking around town, or turning my personal item into a brick of camera +gear to free up room in my carry on. + + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2015/07/nat-geo-mc5350.html --- a/content/blog/2015/07/nat-geo-mc5350.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,470 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "Bag Review: National Geographic MC5350" - snip: "Almost perfect weekend travel and DSLR kit bag." - created: 2015-07-26 13:35:00 - %} - - {% block article %} - -Since I've got a bit of downtime before I move, the bag reviews will continue. -Next up is the National Geographic "Medium Backpack" from the Mediterranean line -(model MG5350). - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-01.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-01.jpg) - -I got this backpack a few months ago from [the Geographic Bags site][natgeo] and -have been using it a lot. If it's out of stock there you can find it [on -Amazon][amazon] (affiliate link). - -I mostly use it for two things: - -* Day hikes where I want to bring a DSLR for landscape photos. -* A weekend bag to fly with as a one-bag carry on. - -It's been almost perfect for these two things, with just a couple of tiny -hitches. Let's dig in! - -[natgeo]: http://www.geographicbags.us/product/85657.1115885.0.0.0/NG%2BMC%2B5350/_/Medium_Backpack_fo_Personal_gear%2C_Laptop%2C_DSLR%2C_acc.%2C -[amazon]: http://www.amazon.com/dp/B00VQT8FYM/?tag=stelos-20 - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-02.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-02.jpg) - -This line (the Mediterranean series) has a much different look than their -others. The branding is straightforward but not over the top. Once again, I'm -pretty sure Nat Geo is licensing the name to Manfrotto to actually make the -bags. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-03.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-03.jpg) - -The stripes are a nice touch that give it some visual interest without being -obnoxious. They remind me of piano keys a bit. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-04.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-04.jpg) - -The front flap has a pocket for small stuff. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-05.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-05.jpg) - -The metal zipper pulls are hefty, and have the logo nicely embossed. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-06.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-06.jpg) - -Some of the zipper pulls are chunky leather. I'm not sure why some are metal -and some are leather, but it works. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-07.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-07.jpg) - -The straps are medium-width, very padded, and not detachable. There's plenty of -room for length adjustments if you're a tall person. Unfortunately there are no -load lifter straps. - -There's no internal frame, but the back of the bag has a lot of padding sewn -into it. The inset compass design gives you a bit of ventilation. It feels -nice when wearing, but also means you can't fold or roll this bag up to stuff it -in a bigger bag to use as a day pack. - -[lx]: http://www.amazon.com/dp/B00GORMJTI/?tag=stelos-20 - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-08.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-08.jpg) - -There's a top handle for yanking it of overhead bins. Also the thin strip on -the back can be pulled out a bit to slip over the handle of some roller luggage, -if you like. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-09.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-09.jpg) - -Handle stitching is nice and sturdy. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-10.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-10.jpg) - -There are a couple of rings on the front of the straps for you to clip things -to. I like using a small carabiner to clip my [Panasonic LX100][lx] to the -right one so I can easily take photos when hiking. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-11.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-11.jpg) - -There's a loop on one of the sides with a small lashing strap. We'll see it in -action later (though I don't tend to use it much). - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-12.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-12.jpg) - -The sternum strap is a simple strap with two D-rings as a buckle. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-13.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-13.jpg) - -It's simple and will never break, but it's a little bit annoying to have to -thread the strap through instead of just using a plastic clip like many other -bags do. - -There's no waist strap on this backpack. It's too short for a waist belt that -would bear load, but it would have been nice to have an optional one to just -secure things a bit better. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-14.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-14.jpg) - -The buckles on the straps bite nicely into the fabric and don't slip. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-15.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-15.jpg) - -Once you open it up, you can see the opening cinches closed with a simple cord. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-16.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-16.jpg) - -Leather bit to keep it closed(ish). - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-17.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-17.jpg) - -Metal to reinforce where the strap holds the fabric. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-18.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-18.jpg) - -The underside of the flap has the same striped pattern. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-19.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-19.jpg) - -The bag is divided into two sections of roughly equal size. The top section has -a few flaps and pockets for minor organization. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-20.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-20.jpg) - -Zipper pouch. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-21.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-21.jpg) - -The flaps don't open very much. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-22.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-22.jpg) - -Anything you put in the would need to be pretty flat (e.g.: a Kindle). - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-23.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-23.jpg) - -The side pockets stick out from the body of the bag. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-24.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-24.jpg) - -A few example objects for scale. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-25.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-25.jpg) - -They're about a big as a standard-sized coffee mug. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-26.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-26.jpg) - -They'll just barely hold a folding set of [Marshall Monitor -headphones][monitor]. - -[monitor]: http://www.amazon.com/dp/B00D3ITOHG/?tag=stelos-20 - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-27.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-27.jpg) - -They'll hold a water bottle, though if it's tall it can be a bit precarious. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-28.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-28.jpg) - -They do work well for holding a monopod, in conjunction with the lashing strap. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-29.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-29.jpg) - -Unfortunately this backpack really can't handle a decent tripod. It's too tall -for the side, and there's no attachment rings you could use to lash it to the -bottom or back. - -This is one thing that does bother me about the bag. A couple of extra D rings -on the side and bottom would solve the tripod carrying problem. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-30.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-30.jpg) - -On to the bottom compartment. This is designed as a DSLR bag, so the bottom is -padded and has dividers like any typical camera backpack. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-31.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-31.jpg) - -It comes with two big, fluffy, padded inserts that are velcroed in. - -It may have come with a third divider that I removed, but if so it's mixed in -with all my other spare dividers and I can't remember. Sorry. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-32.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-32.jpg) - -The camera dividers are actually Manfrotto-braded. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-33.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-33.jpg) - -If you're not a photographer (or just aren't bringing a DSLR), the entire camera -padding bit can come out of the bag. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-34.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-34.jpg) - -It's just held in by a few strips of velcro at the bottom (and a lot of -squishing). So you can have two separate sections for all your stuff. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-35.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-35.jpg) - -But wait: once you take out the camera divider there seems to be a zipper... - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-36.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-36.jpg) - -You can unzip the middle divider and turn the bag into one big space, like -a normal backpack, if you need to carry something big and bulky. I don't do -this much, but it's great to have the option! - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-37.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-37.jpg) - -That's about it for the bag itself. Let's pack it! - -I'm going to pack **a lot** of stuff into it here to give you an idea of the -maximum the bag will hold. I wouldn't normally take all this on a weekend trip, -but if you're a packrat this bag will let you cram a surprising amount into it. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-38.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-38.jpg) - -We'll start with a laptop. There's a separate section for a laptop right -against your back. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-39.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-39.jpg) - -15" Retina Macbook Pro. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-40.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-40.jpg) - -Fits in the laptop pocket with plenty of room to spare. Nat Geo says this bag -will hold a 17" laptop and I believe it. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-41.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-41.jpg) - -The laptop pocket is nicely padded, but if you want even more protection you -could use a laptop sleeve as well. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-42.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-42.jpg) - -Sleeved Macbook will fit. I generally don't bother -- the pocket has enough -padding for me. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-43.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-43.jpg) - -Let's load the DSLR. My full kit is usually a gripped, L-plated Pentax K5 II -and three lenses: 16-50mm, 50-135mm, and 200mm, all f/2.8 and weather-sealed. - -I often leave the 200mm at home to save weight and space. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-44.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-44.jpg) - -It all fits snugly, with one lens attached to the camera. I'm really happy with -this bag because most camera backpacks are either: - -* Designed for Football Dad with a Canon Rebel and kit lens, and don't even come - close to accommodating a DSLR with a battery grip. -* Designed for [Chase Jarvis](http://i.imgur.com/aKeu50l.jpg) with two bodies, - six lenses, and no room for anything else like clothing. - -I really like that this bag lets me carry enough camera for my amateur self, and -then lets me carry other stuff (clothes, rain gear, etc) in the rest of the bag. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-45.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-45.jpg) - -Zips closed nicely. The top prism of the camera produces a bit of a bump (can't -really see it in the photos) but it's not a big deal. If I'm not going to be -using a tripod or monopod I'll remove the L-plate and the bump goes away. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-46.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-46.jpg) - -Time for the rest of the bag! I'll show an example of stuff I'd take on -a weekend trip (though again: I'll lean towards packing too much to show you the -limits of the bag). - -First off: two shirts, two pairs of socks, two pairs of underwear, and hiking -pants. Together with the stuff I wear, that's enough for a weekend. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-47.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-47.jpg) - -I've drank the packing cube Kool-Aid. I use the [Eagle Creek packing -cubes][eagle] and they work pretty well. The smaller two of the set fit well -into this bag -- the large one is too big for it. - -[eagle]: http://www.amazon.com/dp/B00F9S85CS/?tag=stelos-20 - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-48.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-48.jpg) - -Small cube packed with socks and underwear. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-49.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-49.jpg) - -Large cube packed with shirts and pants, with a bit of room left over. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-50.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-50.jpg) - -Add a towel if you're staying in a hostel. This is the [Extra Large Packtowl -Original][towel] which has served me well. I may replace it with their -Ultralight version sometime in the future. - -[towel]: http://www.amazon.com/dp/B0075JJ29E/?tag=stelos-20 - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-51.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-51.jpg) - -Large cube all packed. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-52.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-52.jpg) - -Large cube in the top section of the bag. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-53.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-53.jpg) - -Small cube fits to the side nicely. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-54.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-54.jpg) - -I can't fly without headphones, so they get their own side pocket. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-55.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-55.jpg) - -Easy access once on the plane. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-56.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-56.jpg) - -One thing to watch out for: the side pocket zippers have large, sharp metal -teeth. I could see them ripping fabric if you're not careful. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-57.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-57.jpg) - -On a whim I grabbed this [Utility Kit][kit] when I got the bag and it's proved -to be a handy little organizer. - -[kit]: http://www.geographicbags.us/utility-kit-for-media-accessories-or-travel-items - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-58.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-58.jpg) - -The fabric is thin, but water-resistant. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-59.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-59.jpg) - -It will accommodate quite a bit. For this demo: - -* Rain cover for the backpack -* Cell phone charger -* Toothbrush -* Sleeping mask -* Camera charger -* 77mm polarizer -* Lens cleaning supplies - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-60.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-60.jpg) - -All packed in. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-61.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-61.jpg) - -The center section holds a lot. My friend has fit a travel hair dryer in it. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-62.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-62.jpg) - -The mesh bags are good for small stuff. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-63.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-63.jpg) - -Bottom mesh bit. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-64.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-64.jpg) - -It has a hanger to hang once you're in a hotel room, and also helps hold it -shut. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-65.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-65.jpg) - -Fits snugly on top of the large packing cube. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-66.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-66.jpg) - -Since we've got a rain cover for the backpack, let's add one for ourselves. -I use a [Marmot PreCip][jacket] for hiking (it will also add some warmth in -a pinch). - -[jacket]: http://www.amazon.com/dp/B00I2ZCXQO/?tag=stelos-20 - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-67.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-67.jpg) - -Rolls up nice and small. Apparently I'm out of rubber bands in my house, so -I used the end of an old guitar strap here. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-68.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-68.jpg) - -It will fit (barely) in the other side pocket. Again, mind the zipper teeth as -they feel like they would tear through the thin fabric if you're not careful. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-69.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-69.jpg) - -Just about fully loaded! - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-70.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-70.jpg) - -The cord cinches the top of the bag loosely. It can't close completely though, -so if you're worried about pickpockets... - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-71.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-71.jpg) - -You can loop the strap up... - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-72.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-72.jpg) - -And around, to provide a bit of extra protection. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-73.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-73.jpg) - -Front view. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-74.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-74.jpg) - -The snap on the front is also one of the things I don't like about the bag. -It's magnetic so it snaps itself into place (which is nice), but it's very small -and shallow so it doesn't have a lot of strength. If you pack the bag really -full like this you'll find it coming unsnapped from the strain as you walk. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-75.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-75.jpg) - -A couple of odds and ends for the front pocket. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-76.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-76.jpg) - -Easy access. Don't keep anything important in here. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-77.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-77.jpg) - -Time for the final measurements! This is definitely too big to quality as -a "personal item" but will easy make it as a carry on. United's website says -a carry on should be 22" by 14" by 9". - -We're at 17" wide, which is a bit over 14". You probably won't get called on -it, but if you remove the stuff from the side pockets it collapses down to 13". - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-78.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-78.jpg) - -Thickness-wise we're at 9.5". You can save an inch by taking the laptop out. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-79.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-79.jpg) - -And we're well under the long dimension. - -[![Photo](/media/images{{ parent_url }}/nat-geo-mc5350-80.jpg)](/media/images{{ parent_url }}/full/nat-geo-mc5350-80.jpg) - -That's it! - -All this stuff clocks in at a hefty 24 pounds. Like I said, this was an example -of stuffing the bag to bursting. On a real trip I'd probably leave behind the -computer and charger, camera charger, 200mm lens, and the rain gear. - -After using this on a couple of day hikes and weekend trips I'm really happy -with it. I really like the hybrid "reasonable DSLR kit plus weekend supplies" -style they've taken here. My only nitpicks are: - -* Weak front clasp -* Sharp teeth on the side pouch zippers -* No load lifter straps -* No waist strap -* No D rings to lash things to the bottom or back - -Those are all pretty minor though. I'd definitely recommend this bag if you're -looking for a small, versatile camera backpack. - - {% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2015/07/nat-geo-mc5350.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2015/07/nat-geo-mc5350.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,467 @@ ++++ +title = "Bag Review: National Geographic MC5350" +snip = "Almost perfect weekend travel and DSLR kit bag." +date = 2015-07-26T13:35:00Z +draft = false + ++++ + +Since I've got a bit of downtime before I move, the bag reviews will continue. +Next up is the National Geographic "Medium Backpack" from the Mediterranean line +(model MG5350). + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-01.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-01.jpg) + +I got this backpack a few months ago from [the Geographic Bags site][natgeo] and +have been using it a lot. If it's out of stock there you can find it [on +Amazon][amazon] (affiliate link). + +I mostly use it for two things: + +* Day hikes where I want to bring a DSLR for landscape photos. +* A weekend bag to fly with as a one-bag carry on. + +It's been almost perfect for these two things, with just a couple of tiny +hitches. Let's dig in! + +[natgeo]: http://www.geographicbags.us/product/85657.1115885.0.0.0/NG%2BMC%2B5350/_/Medium_Backpack_fo_Personal_gear%2C_Laptop%2C_DSLR%2C_acc.%2C +[amazon]: http://www.amazon.com/dp/B00VQT8FYM/?tag=stelos-20 + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-02.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-02.jpg) + +This line (the Mediterranean series) has a much different look than their +others. The branding is straightforward but not over the top. Once again, I'm +pretty sure Nat Geo is licensing the name to Manfrotto to actually make the +bags. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-03.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-03.jpg) + +The stripes are a nice touch that give it some visual interest without being +obnoxious. They remind me of piano keys a bit. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-04.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-04.jpg) + +The front flap has a pocket for small stuff. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-05.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-05.jpg) + +The metal zipper pulls are hefty, and have the logo nicely embossed. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-06.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-06.jpg) + +Some of the zipper pulls are chunky leather. I'm not sure why some are metal +and some are leather, but it works. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-07.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-07.jpg) + +The straps are medium-width, very padded, and not detachable. There's plenty of +room for length adjustments if you're a tall person. Unfortunately there are no +load lifter straps. + +There's no internal frame, but the back of the bag has a lot of padding sewn +into it. The inset compass design gives you a bit of ventilation. It feels +nice when wearing, but also means you can't fold or roll this bag up to stuff it +in a bigger bag to use as a day pack. + +[lx]: http://www.amazon.com/dp/B00GORMJTI/?tag=stelos-20 + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-08.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-08.jpg) + +There's a top handle for yanking it of overhead bins. Also the thin strip on +the back can be pulled out a bit to slip over the handle of some roller luggage, +if you like. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-09.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-09.jpg) + +Handle stitching is nice and sturdy. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-10.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-10.jpg) + +There are a couple of rings on the front of the straps for you to clip things +to. I like using a small carabiner to clip my [Panasonic LX100][lx] to the +right one so I can easily take photos when hiking. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-11.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-11.jpg) + +There's a loop on one of the sides with a small lashing strap. We'll see it in +action later (though I don't tend to use it much). + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-12.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-12.jpg) + +The sternum strap is a simple strap with two D-rings as a buckle. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-13.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-13.jpg) + +It's simple and will never break, but it's a little bit annoying to have to +thread the strap through instead of just using a plastic clip like many other +bags do. + +There's no waist strap on this backpack. It's too short for a waist belt that +would bear load, but it would have been nice to have an optional one to just +secure things a bit better. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-14.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-14.jpg) + +The buckles on the straps bite nicely into the fabric and don't slip. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-15.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-15.jpg) + +Once you open it up, you can see the opening cinches closed with a simple cord. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-16.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-16.jpg) + +Leather bit to keep it closed(ish). + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-17.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-17.jpg) + +Metal to reinforce where the strap holds the fabric. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-18.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-18.jpg) + +The underside of the flap has the same striped pattern. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-19.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-19.jpg) + +The bag is divided into two sections of roughly equal size. The top section has +a few flaps and pockets for minor organization. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-20.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-20.jpg) + +Zipper pouch. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-21.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-21.jpg) + +The flaps don't open very much. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-22.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-22.jpg) + +Anything you put in the would need to be pretty flat (e.g.: a Kindle). + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-23.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-23.jpg) + +The side pockets stick out from the body of the bag. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-24.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-24.jpg) + +A few example objects for scale. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-25.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-25.jpg) + +They're about a big as a standard-sized coffee mug. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-26.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-26.jpg) + +They'll just barely hold a folding set of [Marshall Monitor +headphones][monitor]. + +[monitor]: http://www.amazon.com/dp/B00D3ITOHG/?tag=stelos-20 + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-27.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-27.jpg) + +They'll hold a water bottle, though if it's tall it can be a bit precarious. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-28.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-28.jpg) + +They do work well for holding a monopod, in conjunction with the lashing strap. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-29.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-29.jpg) + +Unfortunately this backpack really can't handle a decent tripod. It's too tall +for the side, and there's no attachment rings you could use to lash it to the +bottom or back. + +This is one thing that does bother me about the bag. A couple of extra D rings +on the side and bottom would solve the tripod carrying problem. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-30.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-30.jpg) + +On to the bottom compartment. This is designed as a DSLR bag, so the bottom is +padded and has dividers like any typical camera backpack. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-31.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-31.jpg) + +It comes with two big, fluffy, padded inserts that are velcroed in. + +It may have come with a third divider that I removed, but if so it's mixed in +with all my other spare dividers and I can't remember. Sorry. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-32.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-32.jpg) + +The camera dividers are actually Manfrotto-braded. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-33.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-33.jpg) + +If you're not a photographer (or just aren't bringing a DSLR), the entire camera +padding bit can come out of the bag. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-34.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-34.jpg) + +It's just held in by a few strips of velcro at the bottom (and a lot of +squishing). So you can have two separate sections for all your stuff. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-35.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-35.jpg) + +But wait: once you take out the camera divider there seems to be a zipper... + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-36.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-36.jpg) + +You can unzip the middle divider and turn the bag into one big space, like +a normal backpack, if you need to carry something big and bulky. I don't do +this much, but it's great to have the option! + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-37.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-37.jpg) + +That's about it for the bag itself. Let's pack it! + +I'm going to pack **a lot** of stuff into it here to give you an idea of the +maximum the bag will hold. I wouldn't normally take all this on a weekend trip, +but if you're a packrat this bag will let you cram a surprising amount into it. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-38.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-38.jpg) + +We'll start with a laptop. There's a separate section for a laptop right +against your back. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-39.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-39.jpg) + +15" Retina Macbook Pro. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-40.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-40.jpg) + +Fits in the laptop pocket with plenty of room to spare. Nat Geo says this bag +will hold a 17" laptop and I believe it. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-41.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-41.jpg) + +The laptop pocket is nicely padded, but if you want even more protection you +could use a laptop sleeve as well. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-42.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-42.jpg) + +Sleeved Macbook will fit. I generally don't bother — the pocket has enough +padding for me. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-43.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-43.jpg) + +Let's load the DSLR. My full kit is usually a gripped, L-plated Pentax K5 II +and three lenses: 16-50mm, 50-135mm, and 200mm, all f/2.8 and weather-sealed. + +I often leave the 200mm at home to save weight and space. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-44.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-44.jpg) + +It all fits snugly, with one lens attached to the camera. I'm really happy with +this bag because most camera backpacks are either: + +* Designed for Football Dad with a Canon Rebel and kit lens, and don't even come + close to accommodating a DSLR with a battery grip. +* Designed for [Chase Jarvis](http://i.imgur.com/aKeu50l.jpg) with two bodies, + six lenses, and no room for anything else like clothing. + +I really like that this bag lets me carry enough camera for my amateur self, and +then lets me carry other stuff (clothes, rain gear, etc) in the rest of the bag. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-45.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-45.jpg) + +Zips closed nicely. The top prism of the camera produces a bit of a bump (can't +really see it in the photos) but it's not a big deal. If I'm not going to be +using a tripod or monopod I'll remove the L-plate and the bump goes away. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-46.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-46.jpg) + +Time for the rest of the bag! I'll show an example of stuff I'd take on +a weekend trip (though again: I'll lean towards packing too much to show you the +limits of the bag). + +First off: two shirts, two pairs of socks, two pairs of underwear, and hiking +pants. Together with the stuff I wear, that's enough for a weekend. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-47.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-47.jpg) + +I've drank the packing cube Kool-Aid. I use the [Eagle Creek packing +cubes][eagle] and they work pretty well. The smaller two of the set fit well +into this bag — the large one is too big for it. + +[eagle]: http://www.amazon.com/dp/B00F9S85CS/?tag=stelos-20 + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-48.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-48.jpg) + +Small cube packed with socks and underwear. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-49.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-49.jpg) + +Large cube packed with shirts and pants, with a bit of room left over. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-50.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-50.jpg) + +Add a towel if you're staying in a hostel. This is the [Extra Large Packtowl +Original][towel] which has served me well. I may replace it with their +Ultralight version sometime in the future. + +[towel]: http://www.amazon.com/dp/B0075JJ29E/?tag=stelos-20 + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-51.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-51.jpg) + +Large cube all packed. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-52.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-52.jpg) + +Large cube in the top section of the bag. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-53.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-53.jpg) + +Small cube fits to the side nicely. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-54.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-54.jpg) + +I can't fly without headphones, so they get their own side pocket. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-55.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-55.jpg) + +Easy access once on the plane. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-56.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-56.jpg) + +One thing to watch out for: the side pocket zippers have large, sharp metal +teeth. I could see them ripping fabric if you're not careful. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-57.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-57.jpg) + +On a whim I grabbed this [Utility Kit][kit] when I got the bag and it's proved +to be a handy little organizer. + +[kit]: http://www.geographicbags.us/utility-kit-for-media-accessories-or-travel-items + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-58.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-58.jpg) + +The fabric is thin, but water-resistant. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-59.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-59.jpg) + +It will accommodate quite a bit. For this demo: + +* Rain cover for the backpack +* Cell phone charger +* Toothbrush +* Sleeping mask +* Camera charger +* 77mm polarizer +* Lens cleaning supplies + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-60.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-60.jpg) + +All packed in. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-61.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-61.jpg) + +The center section holds a lot. My friend has fit a travel hair dryer in it. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-62.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-62.jpg) + +The mesh bags are good for small stuff. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-63.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-63.jpg) + +Bottom mesh bit. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-64.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-64.jpg) + +It has a hanger to hang once you're in a hotel room, and also helps hold it +shut. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-65.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-65.jpg) + +Fits snugly on top of the large packing cube. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-66.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-66.jpg) + +Since we've got a rain cover for the backpack, let's add one for ourselves. +I use a [Marmot PreCip][jacket] for hiking (it will also add some warmth in +a pinch). + +[jacket]: http://www.amazon.com/dp/B00I2ZCXQO/?tag=stelos-20 + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-67.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-67.jpg) + +Rolls up nice and small. Apparently I'm out of rubber bands in my house, so +I used the end of an old guitar strap here. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-68.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-68.jpg) + +It will fit (barely) in the other side pocket. Again, mind the zipper teeth as +they feel like they would tear through the thin fabric if you're not careful. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-69.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-69.jpg) + +Just about fully loaded! + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-70.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-70.jpg) + +The cord cinches the top of the bag loosely. It can't close completely though, +so if you're worried about pickpockets... + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-71.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-71.jpg) + +You can loop the strap up... + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-72.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-72.jpg) + +And around, to provide a bit of extra protection. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-73.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-73.jpg) + +Front view. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-74.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-74.jpg) + +The snap on the front is also one of the things I don't like about the bag. +It's magnetic so it snaps itself into place (which is nice), but it's very small +and shallow so it doesn't have a lot of strength. If you pack the bag really +full like this you'll find it coming unsnapped from the strain as you walk. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-75.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-75.jpg) + +A couple of odds and ends for the front pocket. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-76.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-76.jpg) + +Easy access. Don't keep anything important in here. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-77.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-77.jpg) + +Time for the final measurements! This is definitely too big to quality as +a "personal item" but will easy make it as a carry on. United's website says +a carry on should be 22" by 14" by 9". + +We're at 17" wide, which is a bit over 14". You probably won't get called on +it, but if you remove the stuff from the side pockets it collapses down to 13". + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-78.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-78.jpg) + +Thickness-wise we're at 9.5". You can save an inch by taking the laptop out. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-79.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-79.jpg) + +And we're well under the long dimension. + +[![Photo](/media/images/blog/2015/07/nat-geo-mc5350-80.jpg)](/media/images/blog/2015/07/full/nat-geo-mc5350-80.jpg) + +That's it! + +All this stuff clocks in at a hefty 24 pounds. Like I said, this was an example +of stuffing the bag to bursting. On a real trip I'd probably leave behind the +computer and charger, camera charger, 200mm lens, and the rain gear. + +After using this on a couple of day hikes and weekend trips I'm really happy +with it. I really like the hybrid "reasonable DSLR kit plus weekend supplies" +style they've taken here. My only nitpicks are: + +* Weak front clasp +* Sharp teeth on the side pouch zippers +* No load lifter straps +* No waist strap +* No D rings to lash things to the bottom or back + +Those are all pretty minor though. I'd definitely recommend this bag if you're +looking for a small, versatile camera backpack. + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2015/11/beat-the-data.html --- a/content/blog/2015/11/beat-the-data.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,271 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "Just Beat the Data Out of It" - snip: "Round two of the Bob Ross Twitch chat analysis." - created: 2015-11-30 16:10:00 - %} - - {% block article %} - -[Last week][last-week] we played around with a transcript of the Bob Ross Twitch -chat during the Season 2 marathon. I scraped the chat again last Monday to get -the transcript for the Season 3 marathon, so let's pick up where we left off. - -[last-week]: blog/2015/11/happy-little-words/ - -[TOC] - -## Volume Comparison - -Was this week busier or quieter than last week? - -[![Season 2 and 3 chat volume comparison](/media/images{{ parent_url }}/btd-volume-comparison.png)](/media/images{{ parent_url }}/btd-volume-comparison-large.png) - -Note the separate x axes to line up the start and end times of the logs. Also -two-minute buckets were used to make things a bit cleaner to look at on this -crowded graph (see the y axis label). - -Seems like this was a bit quieter than last week. It's encouraging that the -basic structure looks the same -- this hints that there are some patterns -waiting to be discovered. - -## Spiky N-grams - -Last week we looked at graphs of various ngrams and saw that some of them show -pretty clear patterns. The end of each episode brings a flood of `gg`, and when -Bob's son Steve comes on the show we get a big spike in `steve`: - -[![Plot of "gg" and "steve" unigrams in Season 2](/media/images{{ parent_url }}/btd-s2-ggsteve.png)](/media/images{{ parent_url }}/btd-s2-ggsteve-large.png) - -It's reasonable to expect the same behavior this week. What did we get? - -[![Plot of "gg" and "steve" unigrams in Season 3](/media/images{{ parent_url }}/btd-s3-ggsteve.png)](/media/images{{ parent_url }}/btd-s3-ggsteve-large.png) - -Looks pretty similar! In fact the `steve` plot is even more obvious this week. -And in both cases the second streaming of the season repeats the pattern seen -in the first. - -Each week between the two seasons the channel "hosts" another painter. This -just means that it "pipes through" another streamer's channel so people don't -get bored. - -This week whoever is in charge of picking the guest stream did a shitty job. -After the first showing ended viewers were assaulted with the most loud, -obnoxious manchild on the planet. - -The chat was not pleased: - -[![The Douche-o-Meter™](/media/images{{ parent_url }}/btd-s3-douche.png)](/media/images{{ parent_url }}/btd-s3-douche-large.png) - -Thankfully whoever manages Bob's channel mercy-killed the hosting after 10 -minutes or so, and we enjoyed the blissful silence. - -So we've seen that the rate of certain n-grams have clear patterns. If we're -interested in a particular n-gram that's great -- we can graph it and take -a look. But what if we want to *find* interesting n-grams to look at, without -having to watch the whole marathon (or comb through the logs)? - -## Percentiles - -[Percentiles][] are a really useful measurement in a lot of fields, so let's -take a look at them here. We'll start with a relatively common n-gram like -"the": - -[![Percentile graph of "the" in Season 3](/media/images{{ parent_url }}/btd-s3-percentile-the.png)](/media/images{{ parent_url }}/btd-s3-percentile-the-large.png) - -Here we've got a pretty smooth gradation from the lower percentiles up to the -higher ones. Note that these are rates of `the` per minute, so the value `11` -at `50` means that half of all 2-minute bins recorded had eleven or fewer -instances of `the`. This seems low for English text, but a lot of the messages -in the Bob Ross chat are one or two-word slang -- full sentences are rare. - -If we go back to the normal n-gram plot of `the` we can see that it's not a very -"spiky" word: - -[![Plot of "the" unigram in Season 3](/media/images{{ parent_url }}/btd-s3-the.png)](/media/images{{ parent_url }}/btd-s3-the-large.png) - -Let's look at another common word, `bob`: - -[![Percentile graph of "bob"](/media/images{{ parent_url }}/btd-s3-percentile-bob.png)](/media/images{{ parent_url }}/btd-s3-percentile-bob-large.png) - -Pretty smooth, though it's a little bit steeper at the end (probably because of -the deluge of `hi bob` when an episode starts). N-gram plot for comparison: - -[![Plot of "bob" unigram in Season 3](/media/images{{ parent_url }}/btd-s3-bob.png)](/media/images{{ parent_url }}/btd-s3-bob-large.png) - -What about an n-gram we *know* represents a mostly-unique event, like `steve`? -We would expect the graph of percentiles to look steeper, because the lower and -middle percentiles would be very low and the highest few would skyrocket. - -[![Percentile graph of "steve" in Season 3](/media/images{{ parent_url }}/btd-s3-percentile-steve.png)](/media/images{{ parent_url }}/btd-s3-percentile-steve-large.png) - -[![Plot of "steve" unigram in Season 3](/media/images{{ parent_url }}/btd-s3-steve.png)](/media/images{{ parent_url }}/btd-s3-steve-large.png) - -We've tentatively identified another pattern in the data, but how can it help us -find new interesting terms? - -[percentiles]: https://en.wikipedia.org/wiki/Percentile - -## Spikiness Scores - -If we look at the percentiles for a few known-spiky terms we can see a pattern: - -[![Percentile graph of "steve" in Season 3](/media/images{{ parent_url }}/btd-s3-percentile-steve.png)](/media/images{{ parent_url }}/btd-s3-percentile-steve-large.png) - -[![Percentile graph of "drugs" in Season 3](/media/images{{ parent_url }}/btd-s3-percentile-drugs.png)](/media/images{{ parent_url }}/btd-s3-percentile-drugs-large.png) - -[![Percentile graph of "cringe" in Season 3](/media/images{{ parent_url }}/btd-s3-percentile-cringe.png)](/media/images{{ parent_url }}/btd-s3-percentile-cringe-large.png) - -The top percentile or two have some volume, but it quickly drops away to -nothingness within five or ten percent. So let's try to define a really basic -"spikiness score" that we can work out for all n-grams: - -$$ {\text{Spikiness}}(w) = \frac{P_{100}(w)}{P_{95}(w) + 0.1} $$ - -We'll start by saying that the spikiness score of a word is the value of the -100th percentile for that word, divided by the 95th percentile (plus a small -smoothing factor to avoid division by zero). Let's try some words: - - :::text - the 1.78 - bob 2.39 - steve 4.67 - drugs 30.00 - cringe 60.00 - -This doesn't look too terrible. The words we consider spiky are all scored -higher than the non-spiky ones, but it's not quite there yet. `steve` is rated -pretty low even though we consider it to be spiky. - -When we made our initial formula we arbitrarily picked the 100th and 95th -percentiles out of thin air. What if we choose the 99th and 90th instead? - -$$ {\text{Spikiness}}(w) = \frac{P_{99}(w)}{P_{90}(w) + 0.1} $$ - - :::text - bob 3.56 - the 1.42 - steve 77.27 - cringe 20.00 - drugs 10.00 - -This has changed the scores quite a bit, and now they're more like what we want. -But again, we just picked the two percentiles out of thin air. It would be nice -if we could get a feel for how the choice of percentiles affects our spikiness -scores. Once again, let's turn to gnuplot. We'll generalize our function: - -$$ {\text{Spikiness}}(w, L, U) = \frac{P_{U}(w)}{P_{L}(w) + 0.1} $$ - -And graph it for all the combinations of percentiles for a couple of words we -know: - -[![Spikiness percentile sensitivity plot for "the"](/media/images{{ parent_url }}/btd-ssp-the.png)](/media/images{{ parent_url }}/btd-ssp-the-large.png) - -[![Spikiness percentile sensitivity plot for "bob"](/media/images{{ parent_url }}/btd-ssp-bob.png)](/media/images{{ parent_url }}/btd-ssp-bob-large.png) - -[![Spikiness percentile sensitivity plot for "steve"](/media/images{{ parent_url }}/btd-ssp-steve.png)](/media/images{{ parent_url }}/btd-ssp-steve-large.png) - -[![Spikiness percentile sensitivity plot for "rip devil"](/media/images{{ parent_url }}/btd-ssp-rip__devil.png)](/media/images{{ parent_url }}/btd-ssp-rip__devil-large.png) - -These graphs are approaching the point of being impossible to read, but we can -definitely see a pattern. In the first two graphs (common words) the only way -to get a high spikiness score is to choose our formula's lower percentile to be -*really* low (15th percentile or lower). - -In the second two graphs (spiky words) we can see that the score is high when -the upper percentile is 99th or 100th, and the lower percentile is beneath the -90th (or thereabouts). - -Now that we have a hypothesis let's try a couple more plots to see if it still -holds: - -[![Spikiness percentile sensitivity plot for "gg"](/media/images{{ parent_url }}/btd-ssp-gg.png)](/media/images{{ parent_url }}/btd-ssp-gg-large.png) - -`gg` does come in spikes, but it happens so often that we need to select -a smaller lower percentile if we want it to be considered spiky. Whether we -want to depends on what we're looking for -- if we want *rare* events then we -probably want to exclude it. - -`ruined` get spammed so much that it's certainly not rare, and isn't even -particularly spiky in any way: - -[![Spikiness percentile sensitivity plot for "ruined"](/media/images{{ parent_url }}/btd-ssp-ruined.png)](/media/images{{ parent_url }}/btd-ssp-ruined-large.png) - -[![Plot of "ruined" unigram in Season 3](/media/images{{ parent_url }}/btd-s3-ruined.png)](/media/images{{ parent_url }}/btd-s3-ruined-large.png) - -So it looks like we're at least on a reasonable track here. Let's settle the -100th and 90th for now and see where they lead. - -There's one other addition to our spikiness formula we should make before moving -on: if the 100th percentile of a term is small (e.g. less than 5) then while it -might technically be spiky, we probably don't care about it. So we'll just drop -those on the floor and not really worry about them. - -$$ {\text{Spikiness}}(w) = \begin{cases} 0& {\text{if}}\ P_{100}(w) < 5 \\ \frac{P_{100}(w)}{P_{90}(w) + 0.1}& {\text{otherwise}} \end{cases} $$ - -## Results - -Now that we've got a way to measure a term's spikiness, we can calculate it for -all n-grams and sort to find some interesting ones. Let's try it with bigrams: - - :::text - mouth__noises 680.00 - (__mouth 520.00 - soft__music 480.00 - elevator__music 480.00 - noises__) 470.00 - believe__biblethump 460.00 - cool__elevator 450.00 - soft__rock 390.00 - smooth__soft 390.00 - smooth__jazz 380.00 - relaxing__guitar 360.00 - guitar__music 360.00 - son__of 330.00 - music__) 330.00 - (__soft 330.00 - a__gun 320.00 - (__relaxing 320.00 - big__shaft 300.00 - super__steve 290.00 - jazz__music 280.00 - crazy__day 280.00 - zoop__zoop 270.00 - the__heck 270.00 - (__smooth 260.00 - flat__trees 240.00 - steve__! 220.00 - hi__steve 220.00 - ... - -We can get similar results for unigrams, trigrams, etc. Let's graph a couple of -these highly-spiky terms. Twitch chat definitely loves innuendo: - -[![Plot of vaguely sexual n-grams in Season 3](/media/images{{ parent_url }}/btd-s3-innuendo.png)](/media/images{{ parent_url }}/btd-s3-innuendo-large.png) - -Something new this week was the addition of captions, which sometimes included -things like `(soft music)` and `(mouth noises)`. The chat liked to poke fun at -those: - -[![Plot of "soft music" and "mouth noises" bigrams in Season 3](/media/images{{ parent_url }}/btd-s3-mouthnoises.png)](/media/images{{ parent_url }}/btd-s3-mouthnoises-large.png) - -We can also see some particular elements of paintings: - -[![Plot of subject n-grams in Season 3](/media/images{{ parent_url }}/btd-s3-subjects.png)](/media/images{{ parent_url }}/btd-s3-subjects-large.png) - -The lists aren't perfect. They contain a lot of redundant stuff (e.g. `(soft -music)` produces 3 separate bigrams that are all equally spiky), and there's -a bunch of stuff we don't care about as much. But if you're looking to find -some interesting terms they can at least give you a starting point. - -## Join the Fun - -I'm posting this right as the Season 4 marathon is going live on [the Bob Ross -Twitch channel][brtwitch] If you've got some time feel free to pull up your -comfy computer chair and join a few thousand other people for a relaxing evening -with Bob! - -[brtwitch]: http://twitch.tv/BobRoss - - {% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2015/11/beat-the-data.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2015/11/beat-the-data.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,273 @@ ++++ +title = "Just Beat the Data Out of It" +snip = "Round two of the Bob Ross Twitch chat analysis." +date = 2015-11-30T16:10:00Z +draft = false + ++++ + +[Last week][last-week] we played around with a transcript of the Bob Ross Twitch +chat during the Season 2 marathon. I scraped the chat again last Monday to get +the transcript for the Season 3 marathon, so let's pick up where we left off. + +[last-week]: blog/2015/11/happy-little-words/ + +{{% toc %}} + +## Volume Comparison + +Was this week busier or quieter than last week? + +[![Season 2 and 3 chat volume comparison](/media/images/blog/2015/11/btd-volume-comparison.png)](/media/images/blog/2015/11/btd-volume-comparison-large.png) + +Note the separate x axes to line up the start and end times of the logs. Also +two-minute buckets were used to make things a bit cleaner to look at on this +crowded graph (see the y axis label). + +Seems like this was a bit quieter than last week. It's encouraging that the +basic structure looks the same — this hints that there are some patterns +waiting to be discovered. + +## Spiky N-grams + +Last week we looked at graphs of various ngrams and saw that some of them show +pretty clear patterns. The end of each episode brings a flood of `gg`, and when +Bob's son Steve comes on the show we get a big spike in `steve`: + +[![Plot of "gg" and "steve" unigrams in Season 2](/media/images/blog/2015/11/btd-s2-ggsteve.png)](/media/images/blog/2015/11/btd-s2-ggsteve-large.png) + +It's reasonable to expect the same behavior this week. What did we get? + +[![Plot of "gg" and "steve" unigrams in Season 3](/media/images/blog/2015/11/btd-s3-ggsteve.png)](/media/images/blog/2015/11/btd-s3-ggsteve-large.png) + +Looks pretty similar! In fact the `steve` plot is even more obvious this week. +And in both cases the second streaming of the season repeats the pattern seen +in the first. + +Each week between the two seasons the channel "hosts" another painter. This +just means that it "pipes through" another streamer's channel so people don't +get bored. + +This week whoever is in charge of picking the guest stream did a shitty job. +After the first showing ended viewers were assaulted with the most loud, +obnoxious manchild on the planet. + +The chat was not pleased: + +[![The Douche-o-Meter™](/media/images/blog/2015/11/btd-s3-douche.png)](/media/images/blog/2015/11/btd-s3-douche-large.png) + +Thankfully whoever manages Bob's channel mercy-killed the hosting after 10 +minutes or so, and we enjoyed the blissful silence. + +So we've seen that the rate of certain n-grams have clear patterns. If we're +interested in a particular n-gram that's great — we can graph it and take +a look. But what if we want to *find* interesting n-grams to look at, without +having to watch the whole marathon (or comb through the logs)? + +## Percentiles + +[Percentiles][] are a really useful measurement in a lot of fields, so let's +take a look at them here. We'll start with a relatively common n-gram like +"the": + +[![Percentile graph of "the" in Season 3](/media/images/blog/2015/11/btd-s3-percentile-the.png)](/media/images/blog/2015/11/btd-s3-percentile-the-large.png) + +Here we've got a pretty smooth gradation from the lower percentiles up to the +higher ones. Note that these are rates of `the` per minute, so the value `11` +at `50` means that half of all 2-minute bins recorded had eleven or fewer +instances of `the`. This seems low for English text, but a lot of the messages +in the Bob Ross chat are one or two-word slang — full sentences are rare. + +If we go back to the normal n-gram plot of `the` we can see that it's not a very +"spiky" word: + +[![Plot of "the" unigram in Season 3](/media/images/blog/2015/11/btd-s3-the.png)](/media/images/blog/2015/11/btd-s3-the-large.png) + +Let's look at another common word, `bob`: + +[![Percentile graph of "bob"](/media/images/blog/2015/11/btd-s3-percentile-bob.png)](/media/images/blog/2015/11/btd-s3-percentile-bob-large.png) + +Pretty smooth, though it's a little bit steeper at the end (probably because of +the deluge of `hi bob` when an episode starts). N-gram plot for comparison: + +[![Plot of "bob" unigram in Season 3](/media/images/blog/2015/11/btd-s3-bob.png)](/media/images/blog/2015/11/btd-s3-bob-large.png) + +What about an n-gram we *know* represents a mostly-unique event, like `steve`? +We would expect the graph of percentiles to look steeper, because the lower and +middle percentiles would be very low and the highest few would skyrocket. + +[![Percentile graph of "steve" in Season 3](/media/images/blog/2015/11/btd-s3-percentile-steve.png)](/media/images/blog/2015/11/btd-s3-percentile-steve-large.png) + +[![Plot of "steve" unigram in Season 3](/media/images/blog/2015/11/btd-s3-steve.png)](/media/images/blog/2015/11/btd-s3-steve-large.png) + +We've tentatively identified another pattern in the data, but how can it help us +find new interesting terms? + +[percentiles]: https://en.wikipedia.org/wiki/Percentile + +## Spikiness Scores + +If we look at the percentiles for a few known-spiky terms we can see a pattern: + +[![Percentile graph of "steve" in Season 3](/media/images/blog/2015/11/btd-s3-percentile-steve.png)](/media/images/blog/2015/11/btd-s3-percentile-steve-large.png) + +[![Percentile graph of "drugs" in Season 3](/media/images/blog/2015/11/btd-s3-percentile-drugs.png)](/media/images/blog/2015/11/btd-s3-percentile-drugs-large.png) + +[![Percentile graph of "cringe" in Season 3](/media/images/blog/2015/11/btd-s3-percentile-cringe.png)](/media/images/blog/2015/11/btd-s3-percentile-cringe-large.png) + +The top percentile or two have some volume, but it quickly drops away to +nothingness within five or ten percent. So let's try to define a really basic +"spikiness score" that we can work out for all n-grams: + +
$$ {\text{Spikiness}}(w) = \frac{P_{100}(w)}{P_{95}(w) + 0.1} $$
+ +We'll start by saying that the spikiness score of a word is the value of the +100th percentile for that word, divided by the 95th percentile (plus a small +smoothing factor to avoid division by zero). Let's try some words: + +```text +the 1.78 +bob 2.39 +steve 4.67 +drugs 30.00 +cringe 60.00 +``` + +This doesn't look too terrible. The words we consider spiky are all scored +higher than the non-spiky ones, but it's not quite there yet. `steve` is rated +pretty low even though we consider it to be spiky. + +When we made our initial formula we arbitrarily picked the 100th and 95th +percentiles out of thin air. What if we choose the 99th and 90th instead? + +
$$ {\text{Spikiness}}(w) = \frac{P_{99}(w)}{P_{90}(w) + 0.1} $$
+ +```text +bob 3.56 +the 1.42 +steve 77.27 +cringe 20.00 +drugs 10.00 +``` + +This has changed the scores quite a bit, and now they're more like what we want. +But again, we just picked the two percentiles out of thin air. It would be nice +if we could get a feel for how the choice of percentiles affects our spikiness +scores. Once again, let's turn to gnuplot. We'll generalize our function: + +
$$ {\text{Spikiness}}(w, L, U) = \frac{P_{U}(w)}{P_{L}(w) + 0.1} $$
+ +And graph it for all the combinations of percentiles for a couple of words we +know: + +[![Spikiness percentile sensitivity plot for "the"](/media/images/blog/2015/11/btd-ssp-the.png)](/media/images/blog/2015/11/btd-ssp-the-large.png) + +[![Spikiness percentile sensitivity plot for "bob"](/media/images/blog/2015/11/btd-ssp-bob.png)](/media/images/blog/2015/11/btd-ssp-bob-large.png) + +[![Spikiness percentile sensitivity plot for "steve"](/media/images/blog/2015/11/btd-ssp-steve.png)](/media/images/blog/2015/11/btd-ssp-steve-large.png) + +[![Spikiness percentile sensitivity plot for "rip devil"](/media/images/blog/2015/11/btd-ssp-rip__devil.png)](/media/images/blog/2015/11/btd-ssp-rip__devil-large.png) + +These graphs are approaching the point of being impossible to read, but we can +definitely see a pattern. In the first two graphs (common words) the only way +to get a high spikiness score is to choose our formula's lower percentile to be +*really* low (15th percentile or lower). + +In the second two graphs (spiky words) we can see that the score is high when +the upper percentile is 99th or 100th, and the lower percentile is beneath the +90th (or thereabouts). + +Now that we have a hypothesis let's try a couple more plots to see if it still +holds: + +[![Spikiness percentile sensitivity plot for "gg"](/media/images/blog/2015/11/btd-ssp-gg.png)](/media/images/blog/2015/11/btd-ssp-gg-large.png) + +`gg` does come in spikes, but it happens so often that we need to select +a smaller lower percentile if we want it to be considered spiky. Whether we +want to depends on what we're looking for — if we want *rare* events then we +probably want to exclude it. + +`ruined` get spammed so much that it's certainly not rare, and isn't even +particularly spiky in any way: + +[![Spikiness percentile sensitivity plot for "ruined"](/media/images/blog/2015/11/btd-ssp-ruined.png)](/media/images/blog/2015/11/btd-ssp-ruined-large.png) + +[![Plot of "ruined" unigram in Season 3](/media/images/blog/2015/11/btd-s3-ruined.png)](/media/images/blog/2015/11/btd-s3-ruined-large.png) + +So it looks like we're at least on a reasonable track here. Let's settle the +100th and 90th for now and see where they lead. + +There's one other addition to our spikiness formula we should make before moving +on: if the 100th percentile of a term is small (e.g. less than 5) then while it +might technically be spiky, we probably don't care about it. So we'll just drop +those on the floor and not really worry about them. + +
$$ + {\text{Spikiness}}(w) = \begin{cases} 0& {\text{if}}\ P_{100}(w) < 5 \\ \frac{P_{100}(w)}{P_{90}(w) + 0.1}& {\text{otherwise}} \end{cases} +$$
+ +## Results + +Now that we've got a way to measure a term's spikiness, we can calculate it for +all n-grams and sort to find some interesting ones. Let's try it with bigrams: + +```text +mouth__noises 680.00 +(__mouth 520.00 +soft__music 480.00 +elevator__music 480.00 +noises__) 470.00 +believe__biblethump 460.00 +cool__elevator 450.00 +soft__rock 390.00 +smooth__soft 390.00 +smooth__jazz 380.00 +relaxing__guitar 360.00 +guitar__music 360.00 +son__of 330.00 +music__) 330.00 +(__soft 330.00 +a__gun 320.00 +(__relaxing 320.00 +big__shaft 300.00 +super__steve 290.00 +jazz__music 280.00 +crazy__day 280.00 +zoop__zoop 270.00 +the__heck 270.00 +(__smooth 260.00 +flat__trees 240.00 +steve__! 220.00 +hi__steve 220.00 +... +``` + +We can get similar results for unigrams, trigrams, etc. Let's graph a couple of +these highly-spiky terms. Twitch chat definitely loves innuendo: + +[![Plot of vaguely sexual n-grams in Season 3](/media/images/blog/2015/11/btd-s3-innuendo.png)](/media/images/blog/2015/11/btd-s3-innuendo-large.png) + +Something new this week was the addition of captions, which sometimes included +things like `(soft music)` and `(mouth noises)`. The chat liked to poke fun at +those: + +[![Plot of "soft music" and "mouth noises" bigrams in Season 3](/media/images/blog/2015/11/btd-s3-mouthnoises.png)](/media/images/blog/2015/11/btd-s3-mouthnoises-large.png) + +We can also see some particular elements of paintings: + +[![Plot of subject n-grams in Season 3](/media/images/blog/2015/11/btd-s3-subjects.png)](/media/images/blog/2015/11/btd-s3-subjects-large.png) + +The lists aren't perfect. They contain a lot of redundant stuff (e.g. `(soft +music)` produces 3 separate bigrams that are all equally spiky), and there's +a bunch of stuff we don't care about as much. But if you're looking to find +some interesting terms they can at least give you a starting point. + +## Join the Fun + +I'm posting this right as the Season 4 marathon is going live on [the Bob Ross +Twitch channel][brtwitch] If you've got some time feel free to pull up your +comfy computer chair and join a few thousand other people for a relaxing evening +with Bob! + +[brtwitch]: http://twitch.tv/BobRoss + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2015/11/happy-little-words.html --- a/content/blog/2015/11/happy-little-words.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,314 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "Happy Little Words" - snip: "Analyzing the Bob Ross Twitch chat." - created: 2015-11-20 18:43:00 - %} - - {% block article %} - -In late October the video game streaming site Twitch.tv [launched "Twitch -Creative"][twitch-creative], essentially giving people permission to stream -non-video game related creative content on the site. To celebrate the launch -they streamed all 403 episodes of [The Joy of Painting with Bob Ross][joy] in -a giant marathon. - -The Bob Ross channel has its own chat room, and it quickly became packed with -folks watching Bob paint. The chat spawned its own memes and conventions within -days, mostly taking gamer slang (e.g. "gg" for "good game") and applying it to -the show (people spam "gg" in the chat whenever Bob finishes a painting). - -Sadly that marathon has ended, but they've kept the dream alive by having ["Bob -Ross Night" on Mondays][mondays]. Every Monday they're going to stream a season -of the show twice (once at a Europe-friendly time and again for American folks). -Last Monday I scraped the Twitch chat during the marathon(s) of Season 2 and -decided to have some fun poking around at the data. - -[twitch-creative]: http://blog.twitch.tv/2015/10/introducing-twitch-creative/ -[joy]: https://en.wikipedia.org/wiki/The_Joy_of_Painting -[mondays]: http://blog.twitch.tv/2015/11/monday-night-is-bob-ross-night/ - -[TOC] - -## Scraping - -Scraping the chat was pretty easy. Twitch has an IRC gateway for chats, so -I just ran an IRC client ([weechat][]) on a VPS and had it log the channel like -any other. Once the marathon finished I just `scp`'ed down the 8mb log and -started working with it. - -First I trimmed both ends to only leave messages from about an hour and a half -before and after the marathons started and ended. So the data I'm going to work -with runs from 2015-11-16 14:30 to 2015-11-17 07:30 (all times are in UTC), -or 17 hours. - -Then I cleaned it up to -remove some of the cruft (status messages from the client and such) and -lowercase everything: - - cat data/raw | grep -E '^[^\t]+\t <' | gsed -e 's/./\L\0/g' > data/log - -Then I made an ugly little Python script to massage the data into something -a bit easier to work with later: - - :::python - import datetime, sys, time - - def datetime_to_epoch(dt): - return int(time.mktime(dt.timetuple())) - - for line in sys.stdin: - timestamp, nick, msg = (s.strip() for s in line.split('\t', 2)) - - timestamp = datetime_to_epoch( - datetime.datetime.strptime(timestamp, '%Y-%m-%d %H:%M:%S')) - - # strip off <>'s - nick = nick[1:-1] - - print(timestamp, nick, msg) - -This results in a file with one message per line, in the format: - - timestamp nick message goes here... - -On a side note: I tried out [Mosh][] for persisting a connection to the server -(instead of using tmux or screen to persist a session) and it worked pretty -well. I might start using it more often. - -[weechat]: https://weechat.org/ -[Mosh]: https://mosh.mit.edu/ - -## Volume - -Now that we've got a nice clean corpus, let's start playing with it! - -The obvious first question: how many messages did people send in total? - - ><((°> cat data/messages | wc -l - 165368 - -That's almost 10,000 messages per hour! And since there were periods of almost -no activity before, between, and after the two marathons it means the rate -*during* them was well over that! - -Who talked the most? - - ><((°> cat data/messages | cuts -f 2 | sort | uniq -c | sort -nr | head -5 - 269 fuscia13 - 259 almightypainter - 239 sabrinamywaifu - 235 roudydogg1 - 201 ionone - -Some talkative folks (though honestly I expected a bit higher numbers here). -[cuts][] is "**cut** on **s**paces" -- a little function I use so I don't have -to type `-d ' '` all the time. - -[cuts]: https://bitbucket.org/sjl/dotfiles/src/default/fish/functions/cuts.fish - -## N-grams - -The chat has spawned a bunch of its own memes and jargon. I made another ugly -Python script to split up messages into [n-grams] so we can analyze them more -easily: - - :::python - import sys - import nltk - - def window(coll, size): - '''Generate a "sliding window" of tuples of size l over coll. - - coll must be sliceable and have a fixed len. - - ''' - coll_len = len(coll) - for i in range(coll_len): - if i + size > coll_len: - break - else: - yield tuple(coll[i:i+size]) - - for line in sys.stdin: - timestamp, nick, msg = line.split(' ', 2) - - n = int(sys.argv[1]) - - for ngram in set(window(nltk.word_tokenize(msg), n)): - print(timestamp, nick, '__'.join(ngram)) - -This lets us easily split a message into unigrams: - - ><((°> echo "1447680000 sjl beat the devil out of it" | python src/split.py 1 - 1447680000 sjl it - 1447680000 sjl the - 1447680000 sjl beat - 1447680000 sjl of - 1447680000 sjl out - 1447680000 sjl devil - -The order of n-grams within a message isn't preserved because the splitting -script uses a `set` to remove duplicate n-grams. I wanted to remove dupes -because it turns out people frequently copy and paste the same word many times -in a single message and I didn't want that to throw off the numbers. - -Bigrams are just as easy -- just change the parameter to `split.py`: - - ><((°> echo "1447680000 sjl beat the devil out of it" | python src/split.py 2 - 1447680000 sjl of__it - 1447680000 sjl the__devil - 1447680000 sjl beat__the - 1447680000 sjl devil__out - 1447680000 sjl out__of - -N-grams are joined with double underscores to make them easier to plot later. - -So what are the most frequent unigrams? - - ><((°> cat data/words | cuts -f3 | sort | uniq -c | sort -nr | head -15 - 19523 bob - 14367 ruined - 11961 kappaross - 11331 ! - 10989 gg - 7666 is - 6592 the - 6305 ? - 6090 i - 5376 biblethump - 5240 devil - 5122 saved - 5075 rip - 4813 it - 4727 a - -Some of these are expected, like "Bob" and stopwords like "is" and "the". - -The chat loves to spam "RUINED" whenever Bob makes a drastic change to the -painting that looks awful at first, and then spam "SAVED" once he applies a bit -more paint and it looks beautiful. This happens frequently with mountains. - -"KappaRoss" and "BibleThump" are [Twitch emotes][kappa] that produce small -images in the chat. - -When Bob cleans his brush he beats it against the leg of the easel to remove the -paint thinner, and he often smiles and says "just beat the devil out of it". It -didn't take long before chat started spamming "RIP DEVIL" every time he cleans -the brush. - -How about the most frequent bigrams and trigrams? - - ><((°> cat data/bigrams | cuts -f3 | sort | uniq -c | sort -nr | head -15 - 3731 rip__devil - 3153 !__! - 2660 bob__ross - 2490 hi__bob - 1844 <__3 - 1838 kappaross__kappaross - 1533 bob__is - 1409 bob__! - 1389 god__bless - 1324 happy__little - 1181 van__dyke - 1093 gg__wp - 1024 is__back - 908 i__believe - 895 ?__? - - ><((°> cat data/trigrams | cuts -f3 | sort | uniq -c | sort -nr | head -15 - 2130 !__!__! - 1368 kappaross__kappaross__kappaross - 678 van__dyke__brown - 617 bob__is__back - 548 ?__?__? - 503 biblethump__biblethump__biblethump - 401 bob__ross__is - 377 hi__bob__! - 376 beat__the__devil - 361 bob__!__! - 331 bob__<__3 - 324 <__3__< - 324 3__<__3 - 303 i__love__you - 302 son__of__a - -Looks like lots of love for Bob and no sympathy for the devil. It also seems -like [Van Dyke Brown][vdb] is Twitch chat's favorite color by a landslide. - -Note that the exact n-grams depend on the tokenization method. I used NLTK's -`word_tokenize` because it was easy and worked pretty well. -`wordpunct_tokenize` also works, but it splits up basic punctuation a bit too -much for my liking (e.g. it turns `bob's` into three tokens `bob`, `'`, and `s`, -where `word_tokenize` produces just `bob` and `'s`). - -[n-grams]: https://en.wikipedia.org/wiki/N-gram -[kappa]: https://fivethirtyeight.com/features/why-a-former-twitch-employee-has-one-of-the-most-reproduced-faces-ever/ -[vdb]: https://www.bobross.com/ProductDetails.asp?ProductCode=VanDykeBrown - -## Graphing - -Pure numbers are interesting, but [can be misleading][aq]. Let's make some -graphs to get a sense of what the data feels like. I'm using [gnuplot][] to -make the graphs. - -What does the overall volume look like? We'll use minute-wide buckets in the -x axis to make the graph a bit easier to read. - -[![Photo](/media/images{{ parent_url }}/hlw-total.png)](/media/images{{ parent_url }}/hlw-total-large.png) - -Can you tell where the two marathons start and end? - -Let's try to identify where episodes start and finish. Chat usually spams "hi -bob" when an episode starts and "gg" when it finishes, so let's plot those. -We'll use 30-second x buckets here because a minute isn't a fine enough -resolution for the events we're looking for. To make it easier to read we'll -just look at the first half of the first marathon. - -[![Photo](/media/images{{ parent_url }}/hlw-higg.png)](/media/images{{ parent_url }}/hlw-higg-large.png) - -This works pretty well! The graph starts with a big spike of "hi bob", then as -each episode finishes we see a (huge) spike of "gg", followed immediately by -a round of "hi bob" as the next episode starts. - -Can we find all the times Bob cleaned his brush? - -[![Photo](/media/images{{ parent_url }}/hlw-ripdevil.png)](/media/images{{ parent_url }}/hlw-ripdevil-large.png) - -Looks like the devil isn't having a very good time. It's encouraging that the -two seasons have roughly the same structure (three main clusters of peaks). - -Note that there are a couple of smaller peaks between the two showings. Twitch -showed another streamer painting between the two marathons, so it's likely that -she cleaned her brush a couple of times and the chat responded. Fewer people -were watching the stream during the break, hence the smaller peaks. - -When did Bob get the most love? We'll use 5-minute x bins here because we just -want a general idea. - -[![Photo](/media/images{{ parent_url }}/hlw-love.png)](/media/images{{ parent_url }}/hlw-love-large.png) - -Lots of love all around, but especially as he signed off at the end. - -One of my favorite moments was when Bob said something about "changing your mind -in mid **stream**" and the chat started spamming conspiracy theories about how -he somehow knew about the stream 30 years in the past: - -[![Photo](/media/images{{ parent_url }}/hlw-heknew.png)](/media/images{{ parent_url }}/hlw-heknew-large.png) - -[aq]: https://en.wikipedia.org/wiki/Anscombe%27s_quartet -[gnuplot]: http://www.gnuplot.info/ - -## Up Next - -Poking around at this chat corpus was a lot of fun (and *definitely* counts as -studying for my NLP final, *definitely*). I'll probably record the chat during -next week's marathon and do some more poking, specifically around finding unique -events (e.g. his son Steve coming on the show) by comparing rate percentiles. - -If you've got other ideas for things I should graph, [let me know][twitter]. - -[twitter]: http://twitter.com/stevelosh - - {% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2015/11/happy-little-words.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2015/11/happy-little-words.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,313 @@ ++++ +title = "Happy Little Words" +snip = "Analyzing the Bob Ross Twitch chat." +date = 2015-11-20T18:43:00Z +draft = false + ++++ + +In late October the video game streaming site Twitch.tv [launched "Twitch +Creative"][twitch-creative], essentially giving people permission to stream +non-video game related creative content on the site. To celebrate the launch +they streamed all 403 episodes of [The Joy of Painting with Bob Ross][joy] in +a giant marathon. + +The Bob Ross channel has its own chat room, and it quickly became packed with +folks watching Bob paint. The chat spawned its own memes and conventions within +days, mostly taking gamer slang (e.g. "gg" for "good game") and applying it to +the show (people spam "gg" in the chat whenever Bob finishes a painting). + +Sadly that marathon has ended, but they've kept the dream alive by having ["Bob +Ross Night" on Mondays][mondays]. Every Monday they're going to stream a season +of the show twice (once at a Europe-friendly time and again for American folks). +Last Monday I scraped the Twitch chat during the marathon(s) of Season 2 and +decided to have some fun poking around at the data. + +[twitch-creative]: http://blog.twitch.tv/2015/10/introducing-twitch-creative/ +[joy]: https://en.wikipedia.org/wiki/The_Joy_of_Painting +[mondays]: http://blog.twitch.tv/2015/11/monday-night-is-bob-ross-night/ + +{{% toc %}} + +## Scraping + +Scraping the chat was pretty easy. Twitch has an IRC gateway for chats, so +I just ran an IRC client ([weechat][]) on a VPS and had it log the channel like +any other. Once the marathon finished I just `scp`'ed down the 8mb log and +started working with it. + +First I trimmed both ends to only leave messages from about an hour and a half +before and after the marathons started and ended. So the data I'm going to work +with runs from 2015-11-16 14:30 to 2015-11-17 07:30 (all times are in UTC), +or 17 hours. + +Then I cleaned it up to +remove some of the cruft (status messages from the client and such) and +lowercase everything: + + cat data/raw | grep -E '^[^\t]+\t <' | gsed -e 's/./\L\0/g' > data/log + +Then I made an ugly little Python script to massage the data into something +a bit easier to work with later: + +```python +import datetime, sys, time + +def datetime_to_epoch(dt): + return int(time.mktime(dt.timetuple())) + +for line in sys.stdin: + timestamp, nick, msg = (s.strip() for s in line.split('\t', 2)) + + timestamp = datetime_to_epoch( + datetime.datetime.strptime(timestamp, '%Y-%m-%d %H:%M:%S')) + + # strip off <>'s + nick = nick[1:-1] + + print(timestamp, nick, msg) +``` + +This results in a file with one message per line, in the format: + + timestamp nick message goes here... + +On a side note: I tried out [Mosh][] for persisting a connection to the server +(instead of using tmux or screen to persist a session) and it worked pretty +well. I might start using it more often. + +[weechat]: https://weechat.org/ +[Mosh]: https://mosh.mit.edu/ + +## Volume + +Now that we've got a nice clean corpus, let's start playing with it! + +The obvious first question: how many messages did people send in total? + + ><((°> cat data/messages | wc -l + 165368 + +That's almost 10,000 messages per hour! And since there were periods of almost +no activity before, between, and after the two marathons it means the rate +*during* them was well over that! + +Who talked the most? + + ><((°> cat data/messages | cuts -f 2 | sort | uniq -c | sort -nr | head -5 + 269 fuscia13 + 259 almightypainter + 239 sabrinamywaifu + 235 roudydogg1 + 201 ionone + +Some talkative folks (though honestly I expected a bit higher numbers here). +[cuts][] is "**cut** on **s**paces" — a little function I use so I don't have +to type `-d ' '` all the time. + +[cuts]: https://bitbucket.org/sjl/dotfiles/src/default/fish/functions/cuts.fish + +## N-grams + +The chat has spawned a bunch of its own memes and jargon. I made another ugly +Python script to split up messages into [n-grams] so we can analyze them more +easily: + +```python +import sys +import nltk + +def window(coll, size): + '''Generate a "sliding window" of tuples of size l over coll. + + coll must be sliceable and have a fixed len. + + ''' + coll_len = len(coll) + for i in range(coll_len): + if i + size > coll_len: + break + else: + yield tuple(coll[i:i+size]) + +for line in sys.stdin: + timestamp, nick, msg = line.split(' ', 2) + + n = int(sys.argv[1]) + + for ngram in set(window(nltk.word_tokenize(msg), n)): + print(timestamp, nick, '__'.join(ngram)) +``` + +This lets us easily split a message into unigrams: + + ><((°> echo "1447680000 sjl beat the devil out of it" | python src/split.py 1 + 1447680000 sjl it + 1447680000 sjl the + 1447680000 sjl beat + 1447680000 sjl of + 1447680000 sjl out + 1447680000 sjl devil + +The order of n-grams within a message isn't preserved because the splitting +script uses a `set` to remove duplicate n-grams. I wanted to remove dupes +because it turns out people frequently copy and paste the same word many times +in a single message and I didn't want that to throw off the numbers. + +Bigrams are just as easy — just change the parameter to `split.py`: + + ><((°> echo "1447680000 sjl beat the devil out of it" | python src/split.py 2 + 1447680000 sjl of__it + 1447680000 sjl the__devil + 1447680000 sjl beat__the + 1447680000 sjl devil__out + 1447680000 sjl out__of + +N-grams are joined with double underscores to make them easier to plot later. + +So what are the most frequent unigrams? + + ><((°> cat data/words | cuts -f3 | sort | uniq -c | sort -nr | head -15 + 19523 bob + 14367 ruined + 11961 kappaross + 11331 ! + 10989 gg + 7666 is + 6592 the + 6305 ? + 6090 i + 5376 biblethump + 5240 devil + 5122 saved + 5075 rip + 4813 it + 4727 a + +Some of these are expected, like "Bob" and stopwords like "is" and "the". + +The chat loves to spam "RUINED" whenever Bob makes a drastic change to the +painting that looks awful at first, and then spam "SAVED" once he applies a bit +more paint and it looks beautiful. This happens frequently with mountains. + +"KappaRoss" and "BibleThump" are [Twitch emotes][kappa] that produce small +images in the chat. + +When Bob cleans his brush he beats it against the leg of the easel to remove the +paint thinner, and he often smiles and says "just beat the devil out of it". It +didn't take long before chat started spamming "RIP DEVIL" every time he cleans +the brush. + +How about the most frequent bigrams and trigrams? + + ><((°> cat data/bigrams | cuts -f3 | sort | uniq -c | sort -nr | head -15 + 3731 rip__devil + 3153 !__! + 2660 bob__ross + 2490 hi__bob + 1844 <__3 + 1838 kappaross__kappaross + 1533 bob__is + 1409 bob__! + 1389 god__bless + 1324 happy__little + 1181 van__dyke + 1093 gg__wp + 1024 is__back + 908 i__believe + 895 ?__? + + ><((°> cat data/trigrams | cuts -f3 | sort | uniq -c | sort -nr | head -15 + 2130 !__!__! + 1368 kappaross__kappaross__kappaross + 678 van__dyke__brown + 617 bob__is__back + 548 ?__?__? + 503 biblethump__biblethump__biblethump + 401 bob__ross__is + 377 hi__bob__! + 376 beat__the__devil + 361 bob__!__! + 331 bob__<__3 + 324 <__3__< + 324 3__<__3 + 303 i__love__you + 302 son__of__a + +Looks like lots of love for Bob and no sympathy for the devil. It also seems +like [Van Dyke Brown][vdb] is Twitch chat's favorite color by a landslide. + +Note that the exact n-grams depend on the tokenization method. I used NLTK's +`word_tokenize` because it was easy and worked pretty well. +`wordpunct_tokenize` also works, but it splits up basic punctuation a bit too +much for my liking (e.g. it turns `bob's` into three tokens `bob`, `'`, and `s`, +where `word_tokenize` produces just `bob` and `'s`). + +[n-grams]: https://en.wikipedia.org/wiki/N-gram +[kappa]: https://fivethirtyeight.com/features/why-a-former-twitch-employee-has-one-of-the-most-reproduced-faces-ever/ +[vdb]: https://www.bobross.com/ProductDetails.asp?ProductCode=VanDykeBrown + +## Graphing + +Pure numbers are interesting, but [can be misleading][aq]. Let's make some +graphs to get a sense of what the data feels like. I'm using [gnuplot][] to +make the graphs. + +What does the overall volume look like? We'll use minute-wide buckets in the +x axis to make the graph a bit easier to read. + +[![Photo](/media/images/blog/2015/11/hlw-total.png)](/media/images/blog/2015/11/hlw-total-large.png) + +Can you tell where the two marathons start and end? + +Let's try to identify where episodes start and finish. Chat usually spams "hi +bob" when an episode starts and "gg" when it finishes, so let's plot those. +We'll use 30-second x buckets here because a minute isn't a fine enough +resolution for the events we're looking for. To make it easier to read we'll +just look at the first half of the first marathon. + +[![Photo](/media/images/blog/2015/11/hlw-higg.png)](/media/images/blog/2015/11/hlw-higg-large.png) + +This works pretty well! The graph starts with a big spike of "hi bob", then as +each episode finishes we see a (huge) spike of "gg", followed immediately by +a round of "hi bob" as the next episode starts. + +Can we find all the times Bob cleaned his brush? + +[![Photo](/media/images/blog/2015/11/hlw-ripdevil.png)](/media/images/blog/2015/11/hlw-ripdevil-large.png) + +Looks like the devil isn't having a very good time. It's encouraging that the +two seasons have roughly the same structure (three main clusters of peaks). + +Note that there are a couple of smaller peaks between the two showings. Twitch +showed another streamer painting between the two marathons, so it's likely that +she cleaned her brush a couple of times and the chat responded. Fewer people +were watching the stream during the break, hence the smaller peaks. + +When did Bob get the most love? We'll use 5-minute x bins here because we just +want a general idea. + +[![Photo](/media/images/blog/2015/11/hlw-love.png)](/media/images/blog/2015/11/hlw-love-large.png) + +Lots of love all around, but especially as he signed off at the end. + +One of my favorite moments was when Bob said something about "changing your mind +in mid **stream**" and the chat started spamming conspiracy theories about how +he somehow knew about the stream 30 years in the past: + +[![Photo](/media/images/blog/2015/11/hlw-heknew.png)](/media/images/blog/2015/11/hlw-heknew-large.png) + +[aq]: https://en.wikipedia.org/wiki/Anscombe%27s_quartet +[gnuplot]: http://www.gnuplot.info/ + +## Up Next + +Poking around at this chat corpus was a lot of fun (and *definitely* counts as +studying for my NLP final, *definitely*). I'll probably record the chat during +next week's marathon and do some more poking, specifically around finding unique +events (e.g. his son Steve coming on the show) by comparing rate percentiles. + +If you've got other ideas for things I should graph, [let me know][twitter]. + +[twitter]: http://twitter.com/stevelosh + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2015/12/ludum-dare-34.html --- a/content/blog/2015/12/ludum-dare-34.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,492 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "Ludum Dare 34 Postmortem" - snip: 'I made a "game"!' - created: 2015-12-15 16:30:00 - %} - - {% block article %} - -This past weekend was [Ludum Dare 34][]. Ludum Dare is a thrice-a-year event -where a theme is chosen and people have 48 hours (for the competition) or 72 -hours (for the "casual" jam) to create a game based on a theme. - -I actually managed to have some free time for a change, so I decided to give it -a go. I've got a lot of experience programming and a little bit in "sprinting" -like this (my team [placed second in Django Dash][Django Dash] a few years ago), -but I haven't made very many games. Still, by the end of the jam I managed to -make something that's not quite a game but *is* pretty fun to play around with. -I figured I'd write about the process while it's all still fresh in my mind. - -[Ludum Dare 34]: http://ludumdare.com/compo/ -[Django Dash]: http://djangodash.com/judging/c2/results/team/49/ - -[TOC] - -## My Game - -The theme this year was actually *tied* in voting, so there were two: - -* Growing -* Two Button Controls - -### Language & Code - -I knew going in that I wanted to make something with ASCII graphics with Clojure -and [clojure-lanterna][]. Once I thought about the themes I decided to make -a simulation of a world with plants and animals. I settled on the name "Silt" -(the stuff at the bottom of riverbeds that slowly accumulates and reshapes the -world) and got to work. - -You can get the source for the game [on BitBucket][bitbucket]. There's also -a mirror on GitHub if you prefer git. - -### Play - -There's a prebuilt jar file [on BitBucket][bitbucket] if you want to play the -game on your computer. - -If you don't want to run some random code on your machine, I understand. I've -spun up a server you can telnet into to play: - - telnet silt.stevelosh.com - -For best results use a terminal with a dark background. If there are a bunch of -people playing the server may be slow, or fail entirely with out-of-memory -errors. Sorry, it's just an 8gb Linode I'm funding out of my own pocket. - -### Gameplay - -You are the god of a [toroidal][] world. Initially it's inhabited by a small -population of four hundred and one creatures. Time ticks by at a few ticks per -second. - -[![Silt Initial World](/media/images{{ parent_url }}/silt-initial.gif)](/media/images{{ parent_url }}/silt-initial.gif) - -The creatures need energy to survive. They can get energy by eating fruit from -shrubs or being near water. - -If a creature has enough energy it might reproduce by splitting off a clone of -itself. The clone will be identical to its parent (sibling?), though there is -a slight chance of mutation. - -The ideal body temperature for a creature is 0 degrees. The world starts at -that temperature, creating a paradise. As the god of the world, the only way -you can interact with it is to increase or decrease the temperature. - -If the temperature of the world is different than the temperature of a creature, -they will need to spend more energy to maintain their body heat. This makes it -more difficult to survive. Swinging the temperature in large increments quickly -is a sure way to kill off the population. - -The creatures are not entirely at the mercy of the climate, however. Creatures -have an "insulation" score that represents fur or skin. More insulation -protects the creature from the outside climate, at the expense of a small amount -of energy. - -The mechanics of temperature/insulation and reproduction/mutation interact to -cause evolution. If you slowly increase the temperature of the world over a few -thousand ticks, children with more insulation are more likely to survive, and so -your creatures will tend to evolve more insulation over time. - -Creatures' movement patterns are also affected by mutation. Over time your -creatures will likely evolve to wander instead of staying stationary, because -fruit takes time to grow and so it's more effective to keep moving and -gathering. - -Creatures can also change their colors and glyphs through mutation, but this is -much rarer. After letting the game run for a while you'll probably notice -"gangs" of creatures with similar characteristics, all descended from a single -parent. - -[![Silt Later World](/media/images{{ parent_url }}/silt-later.gif)](/media/images{{ parent_url }}/silt-later.gif) - -Finally, there are eight mysterious objects scattered throughout the landscape. -Each one does something, but discovering exactly *what* will be difficult -(unless you read the source). - -### Controls - -The controls are pretty basic: - -* **`arrow keys`** to move your view of the world. -* **`R`** to reset the world. -* **`escape`** to quit the game. - -Put your cursor over a creature to see their stats: - -* **`hjkl`** or **`wasd`** to move your cursor. - -The world ticks along, but you can freeze time: - -* **`space`** to pause/unpause time. - -Those are the basic controls. To actually *interact* with the world you have -only two options (in accordance with the "Two Button Controls" theme): - -* **`+`** to make the world one degree hotter. -* **`-`** to make the world one degree colder. - -[clojure-lanterna]: http://sjl.bitbucket.org/clojure-lanterna -[bitbucket]: https://bitbucket.org/sjl/silt -[toroidal]: https://en.wikipedia.org/wiki/Torus - -## Good Choices - -I managed to make a working, fairly interesting game within the time limit, so -I'm pretty happy with most of my choices. - -### Familiar Environment - -I decided to make Silt in Clojure. If you follow me on [Twitch][] you might -have noticed I've been getting more into Common Lisp these days. I don't really -like writing in Clojure any more, but I used it for the jam because I wanted an -environment I was familiar with. - -This was a good decision. I wouldn't have finished if I tried to use Common -Lisp -- I'd probably still be struggling with whatever its Curses library is. - -A 72-hour jam is not the time to be trying out a new language or framework -you've never used before if you want to have a finished product at the end. -Pick something you know well so you don't spend 30% of your time trying to -Google things. - -[Twitch]: http://twitch.tv/stevelosh - -### Streaming and Talking - -I streamed a bit of the coding process a couple times over the weekend (look in -past broadcasts and they might still be there). I didn't have a ton of viewers, -but it was good to talk things out. Sometimes the stream was my [rubber duck][] -and it helped a bunch. - -[rubber duck]: https://en.wikipedia.org/wiki/Rubber_duck_debugging - -### Releasing Before the End - -Ludum Dare ends at a certain time, but there's a "release hour" afterword when -you can take an hour to package up the final version of your game. I think it's -better if you don't leave it until then to run through your packaging/release -process at least once, though. Once I had the game in a runnable (if not yet -very fun) state, I made it into a jar to make sure that everything works. - -You don't want to find out your build -process is fucked when you only have 40 minutes left, so do a dry run early and -discover any snags while you have lots of time to fix them. - -### Getting Feedback - -Once I had a playable game I asked my friend [Hafdís][] to see if it would run -on Windows. Luckily it did -- Java and Lanterna are pretty nicely -cross-platform. - -She ran the game in the background for a while and showed me the results later. -It's really helpful to get a second opinion about your game (or website or -whatever you're making) as you're working on it. It's too easy to get too close -to your own game and not realize what's fun or painful about it any more. - -Asking in the IRC channel can work, but generally everyone there is busy working -on their own games so you won't get a ton of takers. You can ask someone -online, but I think in-person is better if at all possible. Seeing someone else -enjoy something you've made is a big morale boost, which can be important when -you're dealing with tricky bugs later. - -[Hafdís]: http://havethis.info/ - -### Profiling and Performance - -During the last day I spent an hour or so using a profiler to identify the -hottest spots in the code and improving performance. I didn't focus too much on -speed, but a little bit of attention can go a long way. - -### Ruthless Simplicity - -One of the main reasons I managed to finish despite my lack of experience making -games was being ruthless about making the game as simple as possible. I have -a ton of ideas that I could potentially do, but cramming them all into the game -would take far too long and would probably create a giant mess. - -Focusing on one core mechanic (temperature) and building the things that -interact with it got me pretty far. The fact that the theme was "Two Button -Controls" really helped me a lot with this, because it forced me to come up with -a mechanic simple enough to encode in two buttons. - -### Sleeping - -I was hanging out in the official IRC channel over the weekend and noticed -a bunch of people talking about how tired they were and how little sleep they -were getting. - -Because the time frame is so short there's a tendency to want to stay awake for -as much time as possible to use as many hours as you can. I think this is -usually a mistake. During this weekend, and also for the Django Dash weekend, -I stuck to my normal sleeping/waking schedule and I think it was the right -choice. You may get *more* hours of work in if you don't sleep, but you get -*better* hours of work if you're rested, fed, and showered. - -This is especially tricky if your timezone is offset with the event. The themes -were announced at 2 AM my time, and while I would have loved to be awake and -dive in right away I chose to go to sleep instead and just get started the next -day. A screwed up sleep schedule is almost as bad as not enough sleep (for me -at least). - -## Bad Choices - -I made a few mistakes along the way too (luckily none of them prevented me from -finishing). - -### Clinging to Failed Mechanics - -One of the first mechanics I added to the game was aging -- creatures would age -over time and eventually die of old age. Immediately I had trouble balancing -this against reproduction to avoid population explosions and extinctions, but -I stuck with it for quite a while trying to make it work. - -Eventually I tore the entire aging mechanic out and replaced it with hunger, -which ended up working far, far better. In hindsight I should have abandoned -aging much sooner, as soon as it was obvious that it was making things painful -and unfun. - -### Last-Minute Additions - -I added a couple of mysterious objects right before I packaged up the final -version of the game and I didn't have time to playtest them very much. I tested -that they didn't crash the game, but I assumed their effects would be minor -curiosities and didn't worry too much. - -Now that I've tested it a bit more, they have a bigger effect than I thought -and drastically affect the world. I should have either tested them more or held -off on them until I had time to test. - -### Working Alone - -This one was unavoidable for me this time, but I'm sure Silt could have been -a lot more interesting if I had worked with another programmer, an artist, or -a sound designer. - -## Clojure - -I'll end with a few notes on the experience of writing Silt in Clojure. Like -I said earlier: I don't particularly like Clojure any more, but it was familiar -so I ran with it. After doing a bunch of Common Lisp lately a few things jumped -out at me about writing in Clojure, so I wanted to get them down on paper before -I forget. - -### Destructuring is Nice - -Clojure's ubiquitous destructuring is really nice. Common Lisp has -`destructuring-bind` but it's not as powerful as it is in Clojure, and it's -certainly not threaded through the language nearly as much. - -Destructuring in function arguments is *really* handy -- I almost want to write -a `defun-destructured` macro that will let me do it in CL. - -### Fewer Superfluous Parentheses - -In Clojure's syntax there seems to be a general theme of using fewer -parentheses. I don't just mean using brackets and curly braces for other -literals either. Take the humble `let` in Clojure: - - :::text - (let [x (foo) - y (bar)] - (+ x y)) - -If you translate the vector literal to a Common Lisp list, it would look like -this: - - :::text - (let (x (foo) - y (bar)) - (+ x y)) - -But Common Lisp actually makes you surround the binding pairs with *another* set -of parentheses: - - :::text - (let ((x (foo)) - (y (bar))) - (+ x y)) - -This is annoying. There are always going to be pairs of binding forms, why do -I need to wrap them in extra parens? Clojure's way is cleaner. - -### The Tooling is Good - -Jvisualvm's statistical profiler is a bit homely, but it gets the job done. -It's nice to have mature tools to introspect what your code is doing at runtime. - -`lein uberjar` just worked and produced a jar that runs fine on Windows and OS -X. This was a great relief. I haven't looked at packaging up a Common Lisp app -but I am not optimistic. - -### But the Compiler Is Not Helpful - -SBCL warns me when I'm doing something stupid: - - :::lisp - CL-USER> (defun square (x) (* x x)) - - SQUARE - CL-USER> (defun bad (x y) - (+ x (square y y))) - ; in: DEFUN BAD - ; (SQUARE Y Y) - ; - ; caught STYLE-WARNING: - ; The function was called with two arguments, but wants exactly one. - ; - ; compilation unit finished - ; caught 1 STYLE-WARNING condition - - BAD - CL-USER> - -Clojure happily lets me be stupid without a peep: - - :::clojure - user=> (defn square [x] (* x x)) - #'user/square - user=> (defn bad [x y] - #_=> (+ x (square y y))) - #'user/bad - user=> - -I'm often stupid, so I prefer a compiler that helps prevent it. - -### Errors are Still Terrible - -Clojure stack traces have historically been awful. When I ran into my first -error this time I noticed that the stack trace was... nonexistent? - - :::clojure - user=> (defn foo [] (/ 1 0)) - #'user/foo - user=> (foo) - - ArithmeticException Divide by zero clojure.lang.Numbers.divide (Numbers.java:158) - user=> - -I guess they got tired of people complaining about how the stack traces look -like shit, so they just... removed them? You can get them back with a bit of -work: - - :::clojure - user=> (use '[clojure.stacktrace :only [print-stack-trace]]) - user=> (print-stack-trace *e) - java.lang.ArithmeticException: Divide by zero - at clojure.lang.Numbers.divide (Numbers.java:158) - clojure.lang.Numbers.divide (Numbers.java:3808) - user$foo.invoke (form-init5869611501548592997.clj:1) - user$eval1219.invoke (form-init5869611501548592997.clj:1) - clojure.lang.Compiler.eval (Compiler.java:6782) - clojure.lang.Compiler.eval (Compiler.java:6745) - clojure.core$eval.invoke (core.clj:3081) - clojure.main$repl$read_eval_print__7099$fn__7102.invoke (main.clj:240) - clojure.main$repl$read_eval_print__7099.invoke (main.clj:240) - clojure.main$repl$fn__7108.invoke (main.clj:258) - clojure.main$repl.doInvoke (main.clj:258) - clojure.lang.RestFn.invoke (RestFn.java:1523) - clojure.tools.nrepl.middleware.interruptible_eval$evaluate$fn__623.invoke (interruptible_eval.clj:58) - clojure.lang.AFn.applyToHelper (AFn.java:152) - clojure.lang.AFn.applyTo (AFn.java:144) - clojure.core$apply.invoke (core.clj:630) - clojure.core$with_bindings_STAR_.doInvoke (core.clj:1868) - clojure.lang.RestFn.invoke (RestFn.java:425) - clojure.tools.nrepl.middleware.interruptible_eval$evaluate.invoke (interruptible_eval.clj:56) - clojure.tools.nrepl.middleware.interruptible_eval$interruptible_eval$fn__665$fn__668.invoke (interruptible_eval.clj:191) - clojure.tools.nrepl.middleware.interruptible_eval$run_next$fn__660.invoke (interruptible_eval.clj:159) - clojure.lang.AFn.run (AFn.java:22) - java.util.concurrent.ThreadPoolExecutor.runWorker (ThreadPoolExecutor.java:1142) - java.util.concurrent.ThreadPoolExecutor$Worker.run (ThreadPoolExecutor.java:617) - java.lang.Thread.run (Thread.java:745) - -Oh yeah, *there's* the garbagepile I remember. - -Lisp also hides the traceback at first, but it's easier to get at, and once you -do it's actually *helpful*: - - :::lisp - CL-USER> (declaim (optimize (debug 3))) - - NIL - CL-USER> (defun foo () (/ 1 0)) - ; in: DEFUN FOO - ; (/ 1 0) - ; - ; caught STYLE-WARNING: - ; Lisp error during constant folding: - ; arithmetic error DIVISION-BY-ZERO signalled - ; Operation was /, operands (1 0). - ; - ; compilation unit finished - ; caught 1 STYLE-WARNING condition - - FOO - CL-USER> (foo) - - debugger invoked on a DIVISION-BY-ZERO in thread - #: - arithmetic error DIVISION-BY-ZERO signalled - Operation was /, operands (1 0). - - Type HELP for debugger help, or (SB-EXT:EXIT) to exit from SBCL. - - restarts (invokable by number or by possibly-abbreviated name): - 0: [ABORT] Exit debugger, returning to top level. - - (SB-KERNEL::INTEGER-/-INTEGER 1 0) - 0] backtrace - - Backtrace for: # - 0: (SB-KERNEL::INTEGER-/-INTEGER 1 0) - 1: (/ 1 0) - 2: (FOO) - 3: (SB-INT:SIMPLE-EVAL-IN-LEXENV (FOO) #) - 4: (EVAL (FOO)) - 5: (INTERACTIVE-EVAL (FOO) :EVAL NIL) - 6: (SB-IMPL::REPL-FUN NIL) - 7: ((LAMBDA NIL :IN SB-IMPL::TOPLEVEL-REPL)) - 8: (SB-IMPL::%WITH-REBOUND-IO-SYNTAX #) - 9: (SB-IMPL::TOPLEVEL-REPL NIL) - 10: (SB-IMPL::TOPLEVEL-INIT) - 11: ((FLET #:WITHOUT-INTERRUPTS-BODY-86 :IN SAVE-LISP-AND-DIE)) - 12: ((LABELS SB-IMPL::RESTART-LISP :IN SAVE-LISP-AND-DIE)) - -Notice how it actually warns me *at compile time* about the error, whereas -Clojure merrily compiles it without a peep. Also notice how the SBCL backtrace -gives me actual forms and arguments, instead of a line number (which is probably -inaccurate if you've been editing and eval'ing a file piecemeal). - -### Old Bugs - -Running into [four-year old bugs][clj-700] was fun. - -### RIP Your Heap - -Clojure's data structures and STM system make it pretty easy to write code that -both: - -* Runs correctly -* Generates enormous mountains of garbage - -This can be a good or bad thing, depending on how you look at it and what your -needs are. - -Common Lisp has immutable data structures (via fset) and STM if you want them, -but it also lets you use efficient mutable things just as easily if you want. -It doesn't really encourage one specific way of doing things. - -[clj-700]: http://dev.clojure.org/jira/browse/CLJ-700 - -## Conclusion - -Ludum Dare 34 was a lot of fun. I'm definitely going to mark my calendar and do -the next one too. - -If making a game sounds interesting, you should do it too! You don't need a ton -of programming experience, artistic skills, or anything but a love of video -games and a free weekend. Get out there and make a game! - - {% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2015/12/ludum-dare-34.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2015/12/ludum-dare-34.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,497 @@ ++++ +title = "Ludum Dare 34 Postmortem" +snip = 'I made a "game"!' +date = 2015-12-15T16:30:00Z +draft = false + ++++ + +This past weekend was [Ludum Dare 34][]. Ludum Dare is a thrice-a-year event +where a theme is chosen and people have 48 hours (for the competition) or 72 +hours (for the "casual" jam) to create a game based on a theme. + +I actually managed to have some free time for a change, so I decided to give it +a go. I've got a lot of experience programming and a little bit in "sprinting" +like this (my team [placed second in Django Dash][Django Dash] a few years ago), +but I haven't made very many games. Still, by the end of the jam I managed to +make something that's not quite a game but *is* pretty fun to play around with. +I figured I'd write about the process while it's all still fresh in my mind. + +[Ludum Dare 34]: http://ludumdare.com/compo/ +[Django Dash]: http://djangodash.com/judging/c2/results/team/49/ + +{{% toc %}} + +## My Game + +The theme this year was actually *tied* in voting, so there were two: + +* Growing +* Two Button Controls + +### Language & Code + +I knew going in that I wanted to make something with ASCII graphics with Clojure +and [clojure-lanterna][]. Once I thought about the themes I decided to make +a simulation of a world with plants and animals. I settled on the name "Silt" +(the stuff at the bottom of riverbeds that slowly accumulates and reshapes the +world) and got to work. + +You can get the source for the game [on BitBucket][bitbucket]. There's also +a mirror on GitHub if you prefer git. + +### Play + +There's a prebuilt jar file [on BitBucket][bitbucket] if you want to play the +game on your computer. + +If you don't want to run some random code on your machine, I understand. I've +spun up a server you can telnet into to play: + + telnet silt.stevelosh.com + +For best results use a terminal with a dark background. If there are a bunch of +people playing the server may be slow, or fail entirely with out-of-memory +errors. Sorry, it's just an 8gb Linode I'm funding out of my own pocket. + +### Gameplay + +You are the god of a [toroidal][] world. Initially it's inhabited by a small +population of four hundred and one creatures. Time ticks by at a few ticks per +second. + +[![Silt Initial World](/media/images/blog/2015/12/silt-initial.gif)](/media/images/blog/2015/12/silt-initial.gif) + +The creatures need energy to survive. They can get energy by eating fruit from +shrubs or being near water. + +If a creature has enough energy it might reproduce by splitting off a clone of +itself. The clone will be identical to its parent (sibling?), though there is +a slight chance of mutation. + +The ideal body temperature for a creature is 0 degrees. The world starts at +that temperature, creating a paradise. As the god of the world, the only way +you can interact with it is to increase or decrease the temperature. + +If the temperature of the world is different than the temperature of a creature, +they will need to spend more energy to maintain their body heat. This makes it +more difficult to survive. Swinging the temperature in large increments quickly +is a sure way to kill off the population. + +The creatures are not entirely at the mercy of the climate, however. Creatures +have an "insulation" score that represents fur or skin. More insulation +protects the creature from the outside climate, at the expense of a small amount +of energy. + +The mechanics of temperature/insulation and reproduction/mutation interact to +cause evolution. If you slowly increase the temperature of the world over a few +thousand ticks, children with more insulation are more likely to survive, and so +your creatures will tend to evolve more insulation over time. + +Creatures' movement patterns are also affected by mutation. Over time your +creatures will likely evolve to wander instead of staying stationary, because +fruit takes time to grow and so it's more effective to keep moving and +gathering. + +Creatures can also change their colors and glyphs through mutation, but this is +much rarer. After letting the game run for a while you'll probably notice +"gangs" of creatures with similar characteristics, all descended from a single +parent. + +[![Silt Later World](/media/images/blog/2015/12/silt-later.gif)](/media/images/blog/2015/12/silt-later.gif) + +Finally, there are eight mysterious objects scattered throughout the landscape. +Each one does something, but discovering exactly *what* will be difficult +(unless you read the source). + +### Controls + +The controls are pretty basic: + +* **`arrow keys`** to move your view of the world. +* **`R`** to reset the world. +* **`escape`** to quit the game. + +Put your cursor over a creature to see their stats: + +* **`hjkl`** or **`wasd`** to move your cursor. + +The world ticks along, but you can freeze time: + +* **`space`** to pause/unpause time. + +Those are the basic controls. To actually *interact* with the world you have +only two options (in accordance with the "Two Button Controls" theme): + +* **`+`** to make the world one degree hotter. +* **`-`** to make the world one degree colder. + +[clojure-lanterna]: http://sjl.bitbucket.org/clojure-lanterna +[bitbucket]: https://bitbucket.org/sjl/silt +[toroidal]: https://en.wikipedia.org/wiki/Torus + +## Good Choices + +I managed to make a working, fairly interesting game within the time limit, so +I'm pretty happy with most of my choices. + +### Familiar Environment + +I decided to make Silt in Clojure. If you follow me on [Twitch][] you might +have noticed I've been getting more into Common Lisp these days. I don't really +like writing in Clojure any more, but I used it for the jam because I wanted an +environment I was familiar with. + +This was a good decision. I wouldn't have finished if I tried to use Common +Lisp — I'd probably still be struggling with whatever its Curses library is. + +A 72-hour jam is not the time to be trying out a new language or framework +you've never used before if you want to have a finished product at the end. +Pick something you know well so you don't spend 30% of your time trying to +Google things. + +[Twitch]: http://twitch.tv/stevelosh + +### Streaming and Talking + +I streamed a bit of the coding process a couple times over the weekend (look in +past broadcasts and they might still be there). I didn't have a ton of viewers, +but it was good to talk things out. Sometimes the stream was my [rubber duck][] +and it helped a bunch. + +[rubber duck]: https://en.wikipedia.org/wiki/Rubber_duck_debugging + +### Releasing Before the End + +Ludum Dare ends at a certain time, but there's a "release hour" afterword when +you can take an hour to package up the final version of your game. I think it's +better if you don't leave it until then to run through your packaging/release +process at least once, though. Once I had the game in a runnable (if not yet +very fun) state, I made it into a jar to make sure that everything works. + +You don't want to find out your build +process is fucked when you only have 40 minutes left, so do a dry run early and +discover any snags while you have lots of time to fix them. + +### Getting Feedback + +Once I had a playable game I asked my friend [Hafdís][] to see if it would run +on Windows. Luckily it did — Java and Lanterna are pretty nicely +cross-platform. + +She ran the game in the background for a while and showed me the results later. +It's really helpful to get a second opinion about your game (or website or +whatever you're making) as you're working on it. It's too easy to get too close +to your own game and not realize what's fun or painful about it any more. + +Asking in the IRC channel can work, but generally everyone there is busy working +on their own games so you won't get a ton of takers. You can ask someone +online, but I think in-person is better if at all possible. Seeing someone else +enjoy something you've made is a big morale boost, which can be important when +you're dealing with tricky bugs later. + +[Hafdís]: http://havethis.info/ + +### Profiling and Performance + +During the last day I spent an hour or so using a profiler to identify the +hottest spots in the code and improving performance. I didn't focus too much on +speed, but a little bit of attention can go a long way. + +### Ruthless Simplicity + +One of the main reasons I managed to finish despite my lack of experience making +games was being ruthless about making the game as simple as possible. I have +a ton of ideas that I could potentially do, but cramming them all into the game +would take far too long and would probably create a giant mess. + +Focusing on one core mechanic (temperature) and building the things that +interact with it got me pretty far. The fact that the theme was "Two Button +Controls" really helped me a lot with this, because it forced me to come up with +a mechanic simple enough to encode in two buttons. + +### Sleeping + +I was hanging out in the official IRC channel over the weekend and noticed +a bunch of people talking about how tired they were and how little sleep they +were getting. + +Because the time frame is so short there's a tendency to want to stay awake for +as much time as possible to use as many hours as you can. I think this is +usually a mistake. During this weekend, and also for the Django Dash weekend, +I stuck to my normal sleeping/waking schedule and I think it was the right +choice. You may get *more* hours of work in if you don't sleep, but you get +*better* hours of work if you're rested, fed, and showered. + +This is especially tricky if your timezone is offset with the event. The themes +were announced at 2 AM my time, and while I would have loved to be awake and +dive in right away I chose to go to sleep instead and just get started the next +day. A screwed up sleep schedule is almost as bad as not enough sleep (for me +at least). + +## Bad Choices + +I made a few mistakes along the way too (luckily none of them prevented me from +finishing). + +### Clinging to Failed Mechanics + +One of the first mechanics I added to the game was aging — creatures would age +over time and eventually die of old age. Immediately I had trouble balancing +this against reproduction to avoid population explosions and extinctions, but +I stuck with it for quite a while trying to make it work. + +Eventually I tore the entire aging mechanic out and replaced it with hunger, +which ended up working far, far better. In hindsight I should have abandoned +aging much sooner, as soon as it was obvious that it was making things painful +and unfun. + +### Last-Minute Additions + +I added a couple of mysterious objects right before I packaged up the final +version of the game and I didn't have time to playtest them very much. I tested +that they didn't crash the game, but I assumed their effects would be minor +curiosities and didn't worry too much. + +Now that I've tested it a bit more, they have a bigger effect than I thought +and drastically affect the world. I should have either tested them more or held +off on them until I had time to test. + +### Working Alone + +This one was unavoidable for me this time, but I'm sure Silt could have been +a lot more interesting if I had worked with another programmer, an artist, or +a sound designer. + +## Clojure + +I'll end with a few notes on the experience of writing Silt in Clojure. Like +I said earlier: I don't particularly like Clojure any more, but it was familiar +so I ran with it. After doing a bunch of Common Lisp lately a few things jumped +out at me about writing in Clojure, so I wanted to get them down on paper before +I forget. + +### Destructuring is Nice + +Clojure's ubiquitous destructuring is really nice. Common Lisp has +`destructuring-bind` but it's not as powerful as it is in Clojure, and it's +certainly not threaded through the language nearly as much. + +Destructuring in function arguments is *really* handy — I almost want to write +a `defun-destructured` macro that will let me do it in CL. + +### Fewer Superfluous Parentheses + +In Clojure's syntax there seems to be a general theme of using fewer +parentheses. I don't just mean using brackets and curly braces for other +literals either. Take the humble `let` in Clojure: + +```text +(let [x (foo) + y (bar)] + (+ x y)) +``` + +If you translate the vector literal to a Common Lisp list, it would look like +this: + +```text +(let (x (foo) + y (bar)) + (+ x y)) +``` + +But Common Lisp actually makes you surround the binding pairs with *another* set +of parentheses: + +```text +(let ((x (foo)) + (y (bar))) + (+ x y)) +``` + +This is annoying. There are always going to be pairs of binding forms, why do +I need to wrap them in extra parens? Clojure's way is cleaner. + +### The Tooling is Good + +Jvisualvm's statistical profiler is a bit homely, but it gets the job done. +It's nice to have mature tools to introspect what your code is doing at runtime. + +`lein uberjar` just worked and produced a jar that runs fine on Windows and OS +X. This was a great relief. I haven't looked at packaging up a Common Lisp app +but I am not optimistic. + +### But the Compiler Is Not Helpful + +SBCL warns me when I'm doing something stupid: + +```lisp +CL-USER> (defun square (x) (* x x)) + +SQUARE +CL-USER> (defun bad (x y) +(+ x (square y y))) +; in: DEFUN BAD +; (SQUARE Y Y) +; +; caught STYLE-WARNING: +; The function was called with two arguments, but wants exactly one. +; +; compilation unit finished +; caught 1 STYLE-WARNING condition + +BAD +CL-USER> +``` + +Clojure happily lets me be stupid without a peep: + +```clojure +user=> (defn square [x] (* x x)) +#'user/square +user=> (defn bad [x y] + #_=> (+ x (square y y))) +#'user/bad +user=> +``` + +I'm often stupid, so I prefer a compiler that helps prevent it. + +### Errors are Still Terrible + +Clojure stack traces have historically been awful. When I ran into my first +error this time I noticed that the stack trace was... nonexistent? + +```clojure +user=> (defn foo [] (/ 1 0)) +#'user/foo +user=> (foo) + +ArithmeticException Divide by zero clojure.lang.Numbers.divide (Numbers.java:158) +user=> +``` + +I guess they got tired of people complaining about how the stack traces look +like shit, so they just... removed them? You can get them back with a bit of +work: + +```clojure +user=> (use '[clojure.stacktrace :only [print-stack-trace]]) +user=> (print-stack-trace *e) +java.lang.ArithmeticException: Divide by zero + at clojure.lang.Numbers.divide (Numbers.java:158) + clojure.lang.Numbers.divide (Numbers.java:3808) + user$foo.invoke (form-init5869611501548592997.clj:1) + user$eval1219.invoke (form-init5869611501548592997.clj:1) + clojure.lang.Compiler.eval (Compiler.java:6782) + clojure.lang.Compiler.eval (Compiler.java:6745) + clojure.core$eval.invoke (core.clj:3081) + clojure.main$repl$read_eval_print__7099$fn__7102.invoke (main.clj:240) + clojure.main$repl$read_eval_print__7099.invoke (main.clj:240) + clojure.main$repl$fn__7108.invoke (main.clj:258) + clojure.main$repl.doInvoke (main.clj:258) + clojure.lang.RestFn.invoke (RestFn.java:1523) + clojure.tools.nrepl.middleware.interruptible_eval$evaluate$fn__623.invoke (interruptible_eval.clj:58) + clojure.lang.AFn.applyToHelper (AFn.java:152) + clojure.lang.AFn.applyTo (AFn.java:144) + clojure.core$apply.invoke (core.clj:630) + clojure.core$with_bindings_STAR_.doInvoke (core.clj:1868) + clojure.lang.RestFn.invoke (RestFn.java:425) + clojure.tools.nrepl.middleware.interruptible_eval$evaluate.invoke (interruptible_eval.clj:56) + clojure.tools.nrepl.middleware.interruptible_eval$interruptible_eval$fn__665$fn__668.invoke (interruptible_eval.clj:191) + clojure.tools.nrepl.middleware.interruptible_eval$run_next$fn__660.invoke (interruptible_eval.clj:159) + clojure.lang.AFn.run (AFn.java:22) + java.util.concurrent.ThreadPoolExecutor.runWorker (ThreadPoolExecutor.java:1142) + java.util.concurrent.ThreadPoolExecutor$Worker.run (ThreadPoolExecutor.java:617) + java.lang.Thread.run (Thread.java:745) +``` + +Oh yeah, *there's* the garbagepile I remember. + +Lisp also hides the traceback at first, but it's easier to get at, and once you +do it's actually *helpful*: + +```lisp +CL-USER> (declaim (optimize (debug 3))) + +NIL +CL-USER> (defun foo () (/ 1 0)) +; in: DEFUN FOO +; (/ 1 0) +; +; caught STYLE-WARNING: +; Lisp error during constant folding: +; arithmetic error DIVISION-BY-ZERO signalled +; Operation was /, operands (1 0). +; +; compilation unit finished +; caught 1 STYLE-WARNING condition + +FOO +CL-USER> (foo) + +debugger invoked on a DIVISION-BY-ZERO in thread +#: + arithmetic error DIVISION-BY-ZERO signalled +Operation was /, operands (1 0). + +Type HELP for debugger help, or (SB-EXT:EXIT) to exit from SBCL. + +restarts (invokable by number or by possibly-abbreviated name): + 0: [ABORT] Exit debugger, returning to top level. + +(SB-KERNEL::INTEGER-/-INTEGER 1 0) +0] backtrace + +Backtrace for: # +0: (SB-KERNEL::INTEGER-/-INTEGER 1 0) +1: (/ 1 0) +2: (FOO) +3: (SB-INT:SIMPLE-EVAL-IN-LEXENV (FOO) #) +4: (EVAL (FOO)) +5: (INTERACTIVE-EVAL (FOO) :EVAL NIL) +6: (SB-IMPL::REPL-FUN NIL) +7: ((LAMBDA NIL :IN SB-IMPL::TOPLEVEL-REPL)) +8: (SB-IMPL::%WITH-REBOUND-IO-SYNTAX #) +9: (SB-IMPL::TOPLEVEL-REPL NIL) +10: (SB-IMPL::TOPLEVEL-INIT) +11: ((FLET #:WITHOUT-INTERRUPTS-BODY-86 :IN SAVE-LISP-AND-DIE)) +12: ((LABELS SB-IMPL::RESTART-LISP :IN SAVE-LISP-AND-DIE)) +``` + +Notice how it actually warns me *at compile time* about the error, whereas +Clojure merrily compiles it without a peep. Also notice how the SBCL backtrace +gives me actual forms and arguments, instead of a line number (which is probably +inaccurate if you've been editing and eval'ing a file piecemeal). + +### Old Bugs + +Running into [four-year old bugs][clj-700] was fun. + +### RIP Your Heap + +Clojure's data structures and STM system make it pretty easy to write code that +both: + +* Runs correctly +* Generates enormous mountains of garbage + +This can be a good or bad thing, depending on how you look at it and what your +needs are. + +Common Lisp has immutable data structures (via fset) and STM if you want them, +but it also lets you use efficient mutable things just as easily if you want. +It doesn't really encourage one specific way of doing things. + +[clj-700]: http://dev.clojure.org/jira/browse/CLJ-700 + +## Conclusion + +Ludum Dare 34 was a lot of fun. I'm definitely going to mark my calendar and do +the next one too. + +If making a game sounds interesting, you should do it too! You don't need a ton +of programming experience, artistic skills, or anything but a love of video +games and a free weekend. Get out there and make a game! + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2015/12/permutation-patterns.html --- a/content/blog/2015/12/permutation-patterns.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,349 +0,0 @@ - {% extends "_post.html" %} - - {% load mathjax %} - - {% hyde - title: "What the Hell are Permutation Patterns?" - snip: "A short introduction." - created: 2015-12-10 19:55:00 - %} - - {% block article %} - -I'm currently in the Mathematical Programming class at Reykjavík University and -we're working with permutations and patterns. They're really simple to -understand once they're explained to you, but after searching around online -I haven't found a nice explanation for humans, so here you go. - -None of this is new research, I just want to summarize things so other people -can understand it without having to shovel through the internet trying to piece -together a bunch of terse definitions. All of this is covered in more detail on -Wikipedia, as well as in [Combinatorics of Permutations][book] by Miklós Bóna -(and many other places). - -[book]: http://www.amazon.com/dp/1439850518/?tag=stelos-20 - -[TOC] - -## Permutations (Non-Mathy) - -If you've used permutations in a programming language you can probably skip this -section. - -If you're interested in programming and/or math you've probably heard the term -"permutations" before. It's typically explained to be something like "different -ways to order a collection of objects". - -The key word here is "order" -- permutations are all about ordering. - -### Basics - -Let's look at an example. If you have a list of three cats: - -* Scruffy -* Boots -* Patches - -How many different ways can you list them out? Order matters, and you're also -not allowed to repeat a cat (no cloning here). - - 1 2 3 4 5 6 - Scruffy Scruffy Patches Patches Boots Boots - Boots Patches Scruffy Boots Scruffy Patches - Patches Boots Boots Scruffy Patches Scruffy - - -So there are six permutations. Simple enough. Let's think about the edge -cases. How many ways can you permute one cat? Just one: - - 1 - Scruffy - -Something a bit tougher: how many ways can you permute *zero* cats? This seems -a bit weird, but if you phrase it as "how many ways can you order a list of zero -objects" you can convince yourself the answer is one as well (the single -permutation is "the empty list"). - -We don't want to have to count out the number of permutations by hand every -time, so it would be nice to have a formula. If we have a list of *n* things, -how many different permutations can we come up with? - -We can start by filling the first slot with any item we want, so we have \\(n\\) -options. Then for each of those choices we pick something to go in the next -slot from the \\(n - 1\\) things that are left. We can keep going all the way -down until we're at the last slot, by which point we've only got one thing left, -so we get: - -{% filter mathjax %} - (n)(n - 1)(n - 2)...(1) -{% endfilter %} - -Which is just the [factorial][] of \\(n\\). For our three cats we have: - -{% filter mathjax %} - (3)(3 - 1)(3 - 2) = 3 \cdot 2 \cdot 1 = 6 -{% endfilter %} - -So now we know that: - -{% filter mathjax %} - \operatorname{number-of-permutations}(\mathit{items}) = - \mathit{items}! -{% endfilter %} - -[factorial]: https://en.wikipedia.org/wiki/Factorial - -### Restricting Slots - -Things start to get more interesting when we start to restrict the number of -slots. For example, we can ask "how many different lists of three items can we -take from a group of five items?" - -I'm going to stop using cat names now because it's getting painful to type, so -let's just use letters. Our five items will be: - - a, b, c, d, e - -How many different three-length lists can we produce? - - (a, b, c) (a, b, d) (a, b, e) (a, c, b) (a, c, d) (a, c, e) - (a, d, b) (a, d, c) (a, d, e) (a, e, b) (a, e, c) (a, e, d) - - (b, a, c) (b, a, d) (b, a, e) (b, c, a) (b, c, d) (b, c, e) - (b, d, a) (b, d, c) (b, d, e) (b, e, a) (b, e, c) (b, e, d) - - (c, a, b) (c, a, d) (c, a, e) (c, b, a) (c, b, d) (c, b, e) - (c, d, a) (c, d, b) (c, d, e) (c, e, a) (c, e, b) (c, e, d) - - (d, a, b) (d, a, c) (d, a, e) (d, b, a) (d, b, c) (d, b, e) - (d, c, a) (d, c, b) (d, c, e) (d, e, a) (d, e, b) (d, e, c) - - (e, a, b) (e, a, c) (e, a, d) (e, b, a) (e, b, c) (e, b, d) - (e, c, a) (e, c, b) (e, c, d) (e, d, a) (e, d, b) (e, d, c) - -You can see how working with numerical formulas would be a lot nicer than -listing all those out by hand and counting them. The formula to tell us the -number is: - -{% filter mathjax %} - \operatorname{number-of-permutations}(\mathit{items}, \mathit{slots}) = - \frac - {\mathit{items}!} - {(\mathit{items} - \mathit{slots})!} -{% endfilter %} - -So in this example we have: - -{% filter mathjax %} - \frac - {\mathit{items}!} - {(\mathit{items} - \mathit{slots})!} - = - \frac{5!}{(5 - 3)!} = - \frac{5!}{2!} = - \frac{5 \cdot 4 \cdot 3 \cdot 2 \cdot 1}{2 \cdot 1} = - \frac{120}{2} = - 60 -{% endfilter %} - -Which is correct (feel free to count them by hand if you're skeptical). - -This formula continues to work when the number slots is equal to the number of -elements (like in our cat example). If \\(n = slots = items\\) then: - -{% filter mathjax %} - \frac - {\mathit{items}!} - {(\mathit{items} - \mathit{slots})!} - = - \frac - {\mathit{n}!} - {(\mathit{n} - \mathit{n})!} - = - \frac - {\mathit{n}!} - {0!} - = - \frac - {\mathit{n}!} - {1} - = - \mathit{n}! -{% endfilter %} - -Remember that \\(0!\\) is 1, not 0 as you might first expect. Feel free to plug -in the edge cases we talked about before (zero- and one-length permutations) and -make sure they work. - -## Permutations (Mathy) - -That was a pretty standard introduction to permutations, and it's about as far -as most basic programming/math books go at first (they'll also talk about -"combinations", which are something else that we don't care about right now). -But if we want to start looking at permutations more closely we need to add -a few additional rules. - -So far we've been talking about permutations of arbitrary objects, like cats, -letters, or playing cards. From here on out we're going to restrict ourselves -a bit more: we'll only consider lists of positive integers. This is important -because integers can be compared. We can't say that Scruffy is "less than" or -"greater than" Boots, but we *can* say that 15 is less than 16. - -Notice that we're not including the number zero. When we say "show me all -standard permutations of length 3" we mean: - - 1 2 3 - 1 3 2 - 2 1 3 - 2 3 1 - 3 1 1 - 3 2 1 - -Sorry programmers, permutation people like to count from one. - -You'll notice that I wrote out the permutations above each on their own line, -with the numbers just listed out. This is called "one-line notation". -Sometimes people drop the spaces in between the numbers (e.g. `1 2 3` becomes -`123`) if all the numbers are single digits and they feel like being annoying. - -There's also a "two-line notation" that we won't use here, check out the -Wikipedia page at the end of the post for more. - -### Standard/Classical Permutations - -Another term we'll use is "standard permutations" or "classical patterns" (don't -worry about the word "patterns" too much yet). This just means they contain all -the numbers 1 to \\(n\\) with no missing numbers. So `3 2 4 1` is a standard -permutation (of length 4), but `1 902 23232 5` is not. - -### Subwords/Subsequences - -One more bit of terminology before we get to the meat of the post. A "subword" -of a permutation is all the different lists we can get by dropping none, some, -or all of the items, *without changing the order*. For example, the sublists of -`9 3 1 4` are: - - dropping no items - (9 3 1 4) - - dropping one item - ( 3 1 4) - (9 1 4) - (9 3 4) - (9 3 1 ) - - dropping two items - ( 1 4) - (9 4) - (9 3 ) - ( 3 4) - (9 1 ) - ( 3 1 ) - - dropping three items - (9 ) - ( 3 ) - ( 1 ) - ( 4) - - dropping four items - ( ) - -You will also hear these called "subsequences", but this is way too confusing -because `subsequence` in a programming language almost always means -a *consecutive* portion of a list, so I'm going to stick with "subwords". - -## Permutation Patterns - -Hold onto your hats, things are about to get intense. - -We say that one permutation X **contains** another permutation Y if one of the -subwords of X has **the same length and relative order** as Y. I know that's -confusing and not at all clear, so let's take an example. - -Let X be the permutation `1 4 3 2`. Let Y be `1 2`. Does X contain Y? - -If we look at Y's elements, we see that it starts with the lowest element and -then goes to the highest element. Is there some subword of X, of length 2, that -does the same thing? Sure: `1 4` is a subword of X, has length 2, and starts at -the lowest element then goes to the highest. - -Note that our subword has a `4` in it, but Y only has `1` and `2`. This is the -most confusing part about permutation patterns: we don't care about the actual -numbers themselves -- we only care about the relationships between the numbers. - -Another example. Let X be `1 4 3 2`. Let Y be `2 1`. Does X contain Y? Try -this one on your own before reading on. - -Y is still two elements long, but now it starts at the highest element and goes -the lowest. The subword we used in the last example (`1 4`) doesn't work here, -because it starts low and goes high (the opposite of what we want). But `4 3` -is another subword of X, and that *does* match, so X does contain Y. Note that -`4 2` would also fit here (remember: subwords don't have to be consecutive!). - -Let's try something more interesting. Let X still be `1 4 3 2`, but let's make -Y `1 2 3`. Does X contain Y? - -This one is a little trickier. The order of Y is that it starts at the lowest -element, then it goes to the middle one, then finally it goes to the highest -element. Let's look at the subwords of X that are length 3: - - 1 4 3 - 1 4 2 - 1 3 2 - 4 3 2 - -Do any of those have the same relative ordering as Y? No! None of them start -at the lowest, go to the middle, then go to the highest. So this time X does -not contain Y, so we say that X **avoids** Y. - -## So What? - -So now that we know what "X contains Y" and "X avoids Y" mean, what can we use -this stuff for? I'm not an expert, but I can give you a few examples to whet -your apetite. - -Consider a permutation X that avoids `2 1`. What might X look like? - -Well the empty permutation and any permutation with only one element certainly -avoid `2 1` (they don't have any subwords of the correct length at all). What -about longer ones? - -If X avoids `2 1`, then for any two elements in it, the leftmost element must be -smaller than the rightmost one (otherwise it would *contain* `2 1`). This means -that all the elements must be getting bigger as you go to the right, or to put -it another way: the permutation must be in sorted order! - -Similarly, if X avoids `1 2` then it must be in *reverse* sorted order. Poke at -this for a minute and convince yourself it must be true. Make sure to consider -edge cases like the empty permutation and single-element permutations. - -Another example comes from Knuth: if a permutation contains `2 3 1` it cannot be -sorted by a stack in a single pass. Remember: the numbers in the matching -subword don't have to be consecutive, and they don't have to match the exact -numerals, only the relative order matters. `66 1 99 2 33` contains `2 3 1` -because the subword `66 99 33` has the same relative order (and so `66 1 99 -2 33` cannot be sorted by a stack in a single pass). This is probably not -intuitively obvious, so for further explanation check out Wikipedia, or [this -paper][paper] coincidentally by my professor, or even Knuth's [book][taocp] -itself. - -[paper]: http://www.dmtcs.org/pdfpapers/dmAR0153.pdf -[taocp]: http://www.amazon.com/dp/0201896834/?tag=stelos-20 - -## Further Information - -This was a really quick introduction. I glossed over a bunch of things (like -defining a permutation as a mapping of elements of a set to themselves, the -identity permutation, etc). If you want to dive in more, you can check out -Wikipedia: - -* [Permutations](https://en.wikipedia.org/wiki/Permutation) -* [Permutation Patterns](https://en.wikipedia.org/wiki/Permutation_pattern) - -Or go all-in and grab a copy of [Combinatorics of Permutations][book]. - -I'm also planning on doing another post like this about mesh patterns, which are -a slightly more general/powerful version of these basic patterns. - - {% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2015/12/permutation-patterns.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2015/12/permutation-patterns.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,344 @@ ++++ +title = "What the Hell are Permutation Patterns?" +snip = "A short introduction." +date = 2015-12-10T19:55:00Z +draft = false + ++++ + +I'm currently in the Mathematical Programming class at Reykjavík University and +we're working with permutations and patterns. They're really simple to +understand once they're explained to you, but after searching around online +I haven't found a nice explanation for humans, so here you go. + +None of this is new research, I just want to summarize things so other people +can understand it without having to shovel through the internet trying to piece +together a bunch of terse definitions. All of this is covered in more detail on +Wikipedia, as well as in [Combinatorics of Permutations][book] by Miklós Bóna +(and many other places). + +[book]: http://www.amazon.com/dp/1439850518/?tag=stelos-20 + +{{% toc %}} + +## Permutations (Non-Mathy) + +If you've used permutations in a programming language you can probably skip this +section. + +If you're interested in programming and/or math you've probably heard the term +"permutations" before. It's typically explained to be something like "different +ways to order a collection of objects". + +The key word here is "order" — permutations are all about ordering. + +### Basics + +Let's look at an example. If you have a list of three cats: + +* Scruffy +* Boots +* Patches + +How many different ways can you list them out? Order matters, and you're also +not allowed to repeat a cat (no cloning here). + + 1 2 3 4 5 6 + Scruffy Scruffy Patches Patches Boots Boots + Boots Patches Scruffy Boots Scruffy Patches + Patches Boots Boots Scruffy Patches Scruffy + + +So there are six permutations. Simple enough. Let's think about the edge +cases. How many ways can you permute one cat? Just one: + + 1 + Scruffy + +Something a bit tougher: how many ways can you permute *zero* cats? This seems +a bit weird, but if you phrase it as "how many ways can you order a list of zero +objects" you can convince yourself the answer is one as well (the single +permutation is "the empty list"). + +We don't want to have to count out the number of permutations by hand every +time, so it would be nice to have a formula. If we have a list of *n* things, +how many different permutations can we come up with? + +We can start by filling the first slot with any item we want, so we have \\(n\\) +options. Then for each of those choices we pick something to go in the next +slot from the \\(n - 1\\) things that are left. We can keep going all the way +down until we're at the last slot, by which point we've only got one thing left, +so we get: + +
$$ + (n)(n - 1)(n - 2)...(1) +$$
+ +Which is just the [factorial][] of \\(n\\). For our three cats we have: + +
$$ + (3)(3 - 1)(3 - 2) = 3 \cdot 2 \cdot 1 = 6 +$$
+ +So now we know that: + +
$$ + \operatorname{number-of-permutations}(\mathit{items}) = + \mathit{items}! +$$
+ +[factorial]: https://en.wikipedia.org/wiki/Factorial + +### Restricting Slots + +Things start to get more interesting when we start to restrict the number of +slots. For example, we can ask "how many different lists of three items can we +take from a group of five items?" + +I'm going to stop using cat names now because it's getting painful to type, so +let's just use letters. Our five items will be: + + a, b, c, d, e + +How many different three-length lists can we produce? + + (a, b, c) (a, b, d) (a, b, e) (a, c, b) (a, c, d) (a, c, e) + (a, d, b) (a, d, c) (a, d, e) (a, e, b) (a, e, c) (a, e, d) + + (b, a, c) (b, a, d) (b, a, e) (b, c, a) (b, c, d) (b, c, e) + (b, d, a) (b, d, c) (b, d, e) (b, e, a) (b, e, c) (b, e, d) + + (c, a, b) (c, a, d) (c, a, e) (c, b, a) (c, b, d) (c, b, e) + (c, d, a) (c, d, b) (c, d, e) (c, e, a) (c, e, b) (c, e, d) + + (d, a, b) (d, a, c) (d, a, e) (d, b, a) (d, b, c) (d, b, e) + (d, c, a) (d, c, b) (d, c, e) (d, e, a) (d, e, b) (d, e, c) + + (e, a, b) (e, a, c) (e, a, d) (e, b, a) (e, b, c) (e, b, d) + (e, c, a) (e, c, b) (e, c, d) (e, d, a) (e, d, b) (e, d, c) + +You can see how working with numerical formulas would be a lot nicer than +listing all those out by hand and counting them. The formula to tell us the +number is: + +
$$ + \operatorname{number-of-permutations}(\mathit{items}, \mathit{slots}) = + \frac + {\mathit{items}!} + {(\mathit{items} - \mathit{slots})!} +$$
+ +So in this example we have: + +
$$ + \frac + {\mathit{items}!} + {(\mathit{items} - \mathit{slots})!} + = + \frac{5!}{(5 - 3)!} = + \frac{5!}{2!} = + \frac{5 \cdot 4 \cdot 3 \cdot 2 \cdot 1}{2 \cdot 1} = + \frac{120}{2} = + 60 +$$
+ +Which is correct (feel free to count them by hand if you're skeptical). + +This formula continues to work when the number slots is equal to the number of +elements (like in our cat example). If \\(n = slots = items\\) then: + +
$$ + \frac + {\mathit{items}!} + {(\mathit{items} - \mathit{slots})!} + = + \frac + {\mathit{n}!} + {(\mathit{n} - \mathit{n})!} + = + \frac + {\mathit{n}!} + {0!} + = + \frac + {\mathit{n}!} + {1} + = + \mathit{n}! +$$
+ +Remember that \\(0!\\) is 1, not 0 as you might first expect. Feel free to plug +in the edge cases we talked about before (zero- and one-length permutations) and +make sure they work. + +## Permutations (Mathy) + +That was a pretty standard introduction to permutations, and it's about as far +as most basic programming/math books go at first (they'll also talk about +"combinations", which are something else that we don't care about right now). +But if we want to start looking at permutations more closely we need to add +a few additional rules. + +So far we've been talking about permutations of arbitrary objects, like cats, +letters, or playing cards. From here on out we're going to restrict ourselves +a bit more: we'll only consider lists of positive integers. This is important +because integers can be compared. We can't say that Scruffy is "less than" or +"greater than" Boots, but we *can* say that 15 is less than 16. + +Notice that we're not including the number zero. When we say "show me all +standard permutations of length 3" we mean: + + 1 2 3 + 1 3 2 + 2 1 3 + 2 3 1 + 3 1 1 + 3 2 1 + +Sorry programmers, permutation people like to count from one. + +You'll notice that I wrote out the permutations above each on their own line, +with the numbers just listed out. This is called "one-line notation". +Sometimes people drop the spaces in between the numbers (e.g. `1 2 3` becomes +`123`) if all the numbers are single digits and they feel like being annoying. + +There's also a "two-line notation" that we won't use here, check out the +Wikipedia page at the end of the post for more. + +### Standard/Classical Permutations + +Another term we'll use is "standard permutations" or "classical patterns" (don't +worry about the word "patterns" too much yet). This just means they contain all +the numbers 1 to \\(n\\) with no missing numbers. So `3 2 4 1` is a standard +permutation (of length 4), but `1 902 23232 5` is not. + +### Subwords/Subsequences + +One more bit of terminology before we get to the meat of the post. A "subword" +of a permutation is all the different lists we can get by dropping none, some, +or all of the items, *without changing the order*. For example, the sublists of +`9 3 1 4` are: + + dropping no items + (9 3 1 4) + + dropping one item + ( 3 1 4) + (9 1 4) + (9 3 4) + (9 3 1 ) + + dropping two items + ( 1 4) + (9 4) + (9 3 ) + ( 3 4) + (9 1 ) + ( 3 1 ) + + dropping three items + (9 ) + ( 3 ) + ( 1 ) + ( 4) + + dropping four items + ( ) + +You will also hear these called "subsequences", but this is way too confusing +because `subsequence` in a programming language almost always means +a *consecutive* portion of a list, so I'm going to stick with "subwords". + +## Permutation Patterns + +Hold onto your hats, things are about to get intense. + +We say that one permutation X **contains** another permutation Y if one of the +subwords of X has **the same length and relative order** as Y. I know that's +confusing and not at all clear, so let's take an example. + +Let X be the permutation `1 4 3 2`. Let Y be `1 2`. Does X contain Y? + +If we look at Y's elements, we see that it starts with the lowest element and +then goes to the highest element. Is there some subword of X, of length 2, that +does the same thing? Sure: `1 4` is a subword of X, has length 2, and starts at +the lowest element then goes to the highest. + +Note that our subword has a `4` in it, but Y only has `1` and `2`. This is the +most confusing part about permutation patterns: we don't care about the actual +numbers themselves — we only care about the relationships between the numbers. + +Another example. Let X be `1 4 3 2`. Let Y be `2 1`. Does X contain Y? Try +this one on your own before reading on. + +Y is still two elements long, but now it starts at the highest element and goes +the lowest. The subword we used in the last example (`1 4`) doesn't work here, +because it starts low and goes high (the opposite of what we want). But `4 3` +is another subword of X, and that *does* match, so X does contain Y. Note that +`4 2` would also fit here (remember: subwords don't have to be consecutive!). + +Let's try something more interesting. Let X still be `1 4 3 2`, but let's make +Y `1 2 3`. Does X contain Y? + +This one is a little trickier. The order of Y is that it starts at the lowest +element, then it goes to the middle one, then finally it goes to the highest +element. Let's look at the subwords of X that are length 3: + + 1 4 3 + 1 4 2 + 1 3 2 + 4 3 2 + +Do any of those have the same relative ordering as Y? No! None of them start +at the lowest, go to the middle, then go to the highest. So this time X does +not contain Y, so we say that X **avoids** Y. + +## So What? + +So now that we know what "X contains Y" and "X avoids Y" mean, what can we use +this stuff for? I'm not an expert, but I can give you a few examples to whet +your apetite. + +Consider a permutation X that avoids `2 1`. What might X look like? + +Well the empty permutation and any permutation with only one element certainly +avoid `2 1` (they don't have any subwords of the correct length at all). What +about longer ones? + +If X avoids `2 1`, then for any two elements in it, the leftmost element must be +smaller than the rightmost one (otherwise it would *contain* `2 1`). This means +that all the elements must be getting bigger as you go to the right, or to put +it another way: the permutation must be in sorted order! + +Similarly, if X avoids `1 2` then it must be in *reverse* sorted order. Poke at +this for a minute and convince yourself it must be true. Make sure to consider +edge cases like the empty permutation and single-element permutations. + +Another example comes from Knuth: if a permutation contains `2 3 1` it cannot be +sorted by a stack in a single pass. Remember: the numbers in the matching +subword don't have to be consecutive, and they don't have to match the exact +numerals, only the relative order matters. `66 1 99 2 33` contains `2 3 1` +because the subword `66 99 33` has the same relative order (and so `66 1 99 +2 33` cannot be sorted by a stack in a single pass). This is probably not +intuitively obvious, so for further explanation check out Wikipedia, or [this +paper][paper] coincidentally by my professor, or even Knuth's [book][taocp] +itself. + +[paper]: http://www.dmtcs.org/pdfpapers/dmAR0153.pdf +[taocp]: http://www.amazon.com/dp/0201896834/?tag=stelos-20 + +## Further Information + +This was a really quick introduction. I glossed over a bunch of things (like +defining a permutation as a mapping of elements of a set to themselves, the +identity permutation, etc). If you want to dive in more, you can check out +Wikipedia: + +* [Permutations](https://en.wikipedia.org/wiki/Permutation) +* [Permutation Patterns](https://en.wikipedia.org/wiki/Permutation_pattern) + +Or go all-in and grab a copy of [Combinatorics of Permutations][book]. + +I'm also planning on doing another post like this about mesh patterns, which are +a slightly more general/powerful version of these basic patterns. + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2016/02/midpoint-displacement.html --- a/content/blog/2016/02/midpoint-displacement.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,870 +0,0 @@ - {% extends "_post.html" %} - - {% load mathjax %} - - {% hyde - title: "Terrain Generation with Midpoint Displacement" - snip: "A first step toward growing worlds with computers." - created: 2016-02-19 19:45:00 - %} - - {% block extra_js %} - - - - - {% endblock extra_js %} - - {% block article %} - -I'm taking the Game Engine Architecture class at Reykjavík University. My group -just finished our midterm [project][] where we played around with procedural -terrain generation in [Unity][]. It was a lot of fun and I really enjoyed -creating terrain out of numbers, so I thought I'd write up a little introduction -to a few of the algorithms. - -In this post we'll be looking at Midpoint Displacement. - -The full series of posts so far: - -* [Midpoint Displacement](/blog/2016/02/midpoint-displacement/) -* [Recursive Midpoint Displacement](/blog/2016/03/recursive-midpoint-displacement/) -* [Diamond Square](/blog/2016/06/diamond-square/) - -[project]: https://www.youtube.com/watch?v=G5u79w4qiAA -[Unity]: http://unity3d.com/ - -[TOC] - -## Terrain Generation - -Procedural generation is a pretty big field. The [Wikipedia -article][wikipedia-pg] gives a pretty brief overview. The [Procedural Content -Generation Wiki][pcg-wiki] is an entire wiki devoted to procedural generation. -There's also a small but active [subreddit][r-pg] for interested folks. - -Today we're just going to look at a single algorithm for generating -realistic-looking terrain called "Midpoint Displacement". It's relatively -simple (compared to other algorithms) but it produces terrain that actually -looks kind of cool. - -[wikipedia-pg]: https://en.wikipedia.org/wiki/Procedural_generation -[pcg-wiki]: http://pcg.wikidot.com/ -[r-pg]: https://www.reddit.com/r/proceduralgeneration/ - -## Resources, Code, and Examples - -There are a lot of resources for learning how terrain generation algorithms -work. For midpoint displacement I used the [Wikipedia article][wikipedia-ds], -the [PCG Wiki article][pcgw-midpoint], and academic papers like [this -one][paper]. - -Unfortunately, while there are a *lot* of places that describe algorithms like -this, there are relatively few that explain it *thoroughly*. Academic papers in -particular tend to have a really bad case of -[draw-the-rest-of-the-fucking-owl-syndrome][owl]. They tend to slap an equation -or two on the page and move on without showing any code. - -As anyone who's done much programming knows, there's a big difference between -saying something like "set the midpoints of the diamonds to the average of the -corners" and making a computer actually *do* that. How do we iterate over the -array (row-major or column-major)? Does it matter (yes: if you care about cache -coherency)? What about the edges where there are fewer corners? How random -(and how fast) is our random number generator (and how much do we care)? How do -we write all this code so we can actually read it six months later? - -So I'm going to try to do my part to improve the situation by not just -describing the algorithm, but showing *actual code* that runs and generates -a terrain. - -The full code is [here][code], but we'll be going through the important parts -below. - -[wikipedia-ds]: https://en.wikipedia.org/wiki/Diamond-square_algorithm -[pcgw-midpoint]: http://pcg.wikidot.com/pcg-algorithm:midpoint-displacement-algorithm -[paper]: http://micsymposium.org/mics_2011_proceedings/mics2011_submission_30.pdf -[owl]: https://i.imgur.com/RadSf.jpg -[code]: https://bitbucket.org/sjl/stevelosh/src/default/media/js/terrain1.wisp - -### Wisp - -I'm using a language called [Wisp][wisp]. It's a small, Clojure-inspired -dialect of Lisp that compiles down to vanilla Javascript. - -I went with Javascript because it means I can actually put demos inline in this -post. Sometimes it's so much easier to understand something when you can -actually see it running. - -I've tried to write the code so that it's readable even if you don't know Lisp. -The point of me showing you this code is not to give you something to copy and -paste. The point is that until you *actually write and run* the code, you don't -know all the edge cases and problems that can happen. So by making examples -that actually run I feel a bit more confident that I'm explaining things enough. - -I haven't focused on performance at all. If you want a fast implementation of -this, you'll probably want to use a language with real arrays (i.e. contiguous -chunks of memory holding a bunch of unboxed floats or integers). - -[wisp]: https://github.com/Gozala/wisp - -### Three.js - -[Three.js][three.js] is a Javascript library for 3D rendering. I'm using it -here to get something on the screen you can play with. I'm not going to cover -the code to put the terrains on the screen because it's really an entirely -separate problem to *generating* the terrain, and it's mostly uninteresting -boilerplate for these little demos. - -I haven't been able to get these demos working on my iPhone. Sorry about that, -but I'm not much of a frontend developer. You'll have to use a computer to see -them. [Pull requests][slc] are welcome. - -[slc]: http://bitbucket.org/sjl/stevelosh/ -[three.js]: http://threejs.org/ - -## Heightmaps - -Let's get started. The main data structure we're going to be using is -a [heightmap][]. - -### Overview - -Heightmaps are essentially just big two-dimensional arrays of numbers in a given -range that represent the height of the ground at a given point. For example, -a heightmap with a small "mountain" in the middle might look like: - - :::text - [[0, 0, 2, 0, 0], - [0, 2, 5, 1, 0], - [2, 5, 9, 4, 1], - [0, 1, 4, 1, 0], - [0, 0, 1, 0, 0]] - -In practice, heightmaps are often represented as a single-dimensional array -instead to ensure that all the data is together in one big hunk of memory: - - :::text - [0, 0, 2, 0, 0, - 0, 2, 5, 1, 0, - 2, 5, 9, 4, 1, - 0, 1, 4, 1, 0, - 0, 0, 1, 0, 0] - -The type and range of the numbers in the map varies by program and programmer. -Some programs use nonzero integers, some use all the integers, some use floating -point, etc. - -The heightmaps in this post will contain floats from `0.0` to `1.0`. I like -using floats between zero and one because it's easy to roughly visualize them in -your head (e.g. `0.1` is ten percent of the maximum height). - -One common way to visualize heightmaps is by turning them into greyscale images -with one pixel per number, with black pixels for lowest value in the map's range -and white for the highest. The [Wikipedia article][heightmap] has an example of -this. This is handy because you can also write some code to *read* images, and -then you can use any image editor to draw your own heightmaps by hand. - -We're going to visualize the heightmaps in 3D. It's a lot more fun, and they -start to actually look like real terrains. - -[heightmap]: https://en.wikipedia.org/wiki/Heightmap - -### Code - -We're going to start by writing a few functions that will hide away the internal -details of how our heightmaps are implemented, so we can talk about them in -nice high-level terms for the rest of the program. We'll also need a couple of -helper functions along the way. - -We'll start with something to create a heightmap: - - :::clojure - (defn make-heightmap [exponent] - (let [resolution (+ 1 (Math.pow 2 exponent))] - (l (+ "Creating " resolution " by " resolution " heightmap...")) - (def heightmap - (new Array (* resolution resolution))) - (set! heightmap.resolution resolution) - (set! heightmap.exponent exponent) - (set! heightmap.last (- resolution 1)) - (zero-heightmap heightmap))) - -`make-heightmap` takes an exponent and creates a heightmap. For reasons that -will become clear later, all of our heightmaps must be square, and they'll need -to have \\(2^n + 1\\) rows and columns. This means we can make heightmaps of -sizes like 3x3, 5x5, 9x9, 17x17, 33x33, etc. So our function just takes the -\\(n\\) to use and creates a correctly-sized array. - -(If you've ever poked around with Unity's Terrain objects you might recognize -these "powers of two plus one".) - -We create a new array (1-dimensional) and store a few pieces of extra data on it -for later use. `last` is the index of the last row/column, so for a 3x3 array -it would be `2`. - -`l` is just a simple little logging function: - - :::clojure - (defn l [v] - (console.log v)) - -We'll need to zero out the heightmap as well: - - :::clojure - (defn zero-heightmap [heightmap] - (do-times i heightmap.length - (set! (aget heightmap i) 0.0)) - heightmap) - -`zero-heightmap` just iterates from `0` to `heightmap.length` and sets each -element to `0.0`. Wisp doesn't have syntax to loop from 0 to some number, but -it's a Lisp, so we can make some: - - :::clojure - (defmacro when [condition & body] - `(if ~condition - (do ~@body))) - - (defmacro do-times [varname limit & body] - (let [end (gensym)] - `(let [~end ~limit] - (loop [~varname 0] - (when (< ~varname ~end) - ~@body - (recur (+ 1 ~varname))))))) - -If you're not a Lisp person, don't sweat this too much. Just trust me that -`(do-times i 10 (console.log i))` will print the numbers `0` to `9`. - -Now that we can create a heightmap, we'll probably want to be able to read its -elements: - - :::clojure - (defmacro heightmap-get [hm x y] - `(aget ~hm (+ (* ~y (.-resolution ~hm)) ~x))) - - (defn heightmap-get-safe [hm x y] - (when (and (<= 0 x hm.last) - (<= 0 y hm.last)) - (heightmap-get hm x y))) - -`heightmap-get` handles the nasty business of translating our human-thinkable -`x` and `y` coordinates into an index into the single-dimensional array. It's -not particularly fast (especially if the compiler won't inline it), but we're -not worried about speed for this post. - -`heightmap-get-safe` is a version of `heightmap-get` that will do bounds -checking for us and return `nil` if we ask for something out of range. -Javascript will return `undefined` if you ask for a non-existent index, but -because we're storing the array as one big line of data, using a `y` value -that's too big will actually "wrap around" to the next row, so it's safer to -just check it explicitly. - -Of course we'll also want to be able to set values in our heightmap: - - :::clojure - (defmacro heightmap-set! [hm x y val] - `(set! (heightmap-get ~hm ~x ~y) ~val)) - - -And finally, we'll want to be able to "normalize" our array. During the course -of running our algorithm, it's possible for the values to become less than `0.0` -or greater than `1.0`. We want our heightmaps to contain only values between -zero and one, so rather than change the algorithm we can just loop through the -array at the end and "squeeze" the range down to 0 and 1 (or "stretch" it if -it ends up smaller): - - :::clojure - (defn normalize [hm] - (let [max (- Infinity) - min Infinity] - (do-times i hm.length - (let [el (aget hm i)] - (when (< max el) (set! max el)) - (when (> min el) (set! min el)))) - (let [span (- max min)] - (do-times i hm.length - (set! (aget hm i) - (/ (- (aget hm i) min) - span)))))) - -## Random Noise - -Now that we've got a nice little interface to heightmaps we can move on to -actually making some terrain! Before we dive into midpoint displacement, let's -get a [black triangle][] on the screen. We'll start by just setting every point -in the heightmap to a completely random value. - - :::clojure - (defn rand [] - (Math.random)) - - (defn rand-around-zero [spread] - (- (* spread (rand) 2) spread)) - - (defn random-noise [heightmap] - (do-times i heightmap.length - (set! (aget heightmap i) (rand)))) - -Let's see it! Hit the "refresh" button to create and render the terrain. You -can keep hitting it to get fresh terrains. - -
- -Not really a very fun landscape to explore, but at least we've got something -running. Onward to the real algorithm. - -[black triangle]: http://rampantgames.com/blog/?p=7745 - -## Midpoint Displacement - -Midpoint displacement is a simple, fast way to generate decent-looking terrain. -It's not perfect (you'll notice patterns once you start viewing the mesh from -different angles) but it's a good place to start. - -The general process goes like this: - -1. Initialize the four corners of the heightmap to random values. -2. Set the midpoints of each edge to the average of the two corners it's - between, plus or minus a random amount. -3. Set the center of the square to the average of those edge midpoints you just - set, again with a random jitter. -4. Recurse on the four squares inside this one, reducing the jitter. - -That's the standard description. Now let's draw the fucking owl. - -### Initialize the Corners - -The first step is pretty easy: we just need to take our heightmap: - -
-               Column
-          0   1   2   3   4
-        ┌───┬───┬───┬───┬───┐
-      0 │   │   │   │   │   │
-        ├───┼───┼───┼───┼───┤
-      1 │   │   │   │   │   │
-    R   ├───┼───┼───┼───┼───┤
-    o 2 │   │   │   │   │   │
-    w   ├───┼───┼───┼───┼───┤
-      3 │   │   │   │   │   │
-        ├───┼───┼───┼───┼───┤
-      4 │   │   │   │   │   │
-        └───┴───┴───┴───┴───┘
-
- -Find the corners: - -
-               Column
-          0   1   2   3   4
-        ┌───┬───┬───┬───┬───┐
-      0 │ ◉ │   │   │   │ ◉ │
-        ├───┼───┼───┼───┼───┤
-      1 │   │   │   │   │   │
-    R   ├───┼───┼───┼───┼───┤
-    o 2 │   │   │   │   │   │
-    w   ├───┼───┼───┼───┼───┤
-      3 │   │   │   │   │   │
-        ├───┼───┼───┼───┼───┤
-      4 │ ◉ │   │   │   │ ◉ │
-        └───┴───┴───┴───┴───┘
-
- -And shove random values in them: - -
-               Column
-          0   1   2   3   4
-        ┌───┬───┬───┬───┬───┐
-      0 │ 2 │   │   │   │ 8 │
-        ├───┼───┼───┼───┼───┤
-      1 │   │   │   │   │   │
-    R   ├───┼───┼───┼───┼───┤
-    o 2 │   │   │   │   │   │
-    w   ├───┼───┼───┼───┼───┤
-      3 │   │   │   │   │   │
-        ├───┼───┼───┼───┼───┤
-      4 │ 0 │   │   │   │ 3 │
-        └───┴───┴───┴───┴───┘
-
- -(I'm using 0 to 10 instead of 0 to 1 because it's easier to read in the -ASCII-art diagrams.) - -The code is about what you'd expect: - - :::clojure - (defn mpd-init-corners [heightmap] - (heightmap-set! heightmap 0 0 (rand)) - (heightmap-set! heightmap 0 heightmap.last (rand)) - (heightmap-set! heightmap heightmap.last 0 (rand)) - (heightmap-set! heightmap heightmap.last heightmap.last (rand))) - - (defn midpoint-displacement-d1 [heightmap] - (mpd-init-corners heightmap)) - -That was easy. - -
- -### Set the Edges - -For the next step, we need to take our square heightmap with filled-in corners -and use the corners to fill in the middle element of each edge: - -
-               Column
-          0   1   2   3   4
-        ┌───┬───┬───┬───┬───┐
-      0 │ 2 │   │ ◉ │   │ 8 │
-        ├───┼───┼───┼───┼───┤
-      1 │   │   │   │   │   │
-    R   ├───┼───┼───┼───┼───┤
-    o 2 │ ◉ │   │   │   │ ◉ │
-    w   ├───┼───┼───┼───┼───┤
-      3 │   │   │   │   │   │
-        ├───┼───┼───┼───┼───┤
-      4 │ 0 │   │ ◉ │   │ 3 │
-        └───┴───┴───┴───┴───┘
-
-
-
-               Column
-          0   1   2   3   4
-        ┌───┬───┬───┬───┬───┐
-      0 │ 2 ├───▶ ◉ ◀───┤ 8 │
-        ├─┬─┼───┼───┼───┼─┬─┤
-      1 │ │ │   │   │   │ │ │
-    R   ├─▼─┼───┼───┼───┼─▼─┤
-    o 2 │ ◉ │   │   │   │ ◉ │
-    w   ├─▲─┼───┼───┼───┼─▲─┤
-      3 │ │ │   │   │   │ │ │
-        ├─┴─┼───┼───┼───┼─┴─┤
-      4 │ 0 ├───▶ ◉ ◀───┤ 3 │
-        └───┴───┴───┴───┴───┘
-
-
-               Column
-          0   1   2   3   4
-        ┌───┬───┬───┬───┬───┐
-      0 │ 2 ├───▶ 5 ◀───┤ 8 │
-        ├─┬─┼───┼───┼───┼─┬─┤
-      1 │ │ │   │   │   │ │ │
-    R   ├─▼─┼───┼───┼───┼─▼─┤
-    o 2 │ 1 │   │   │   │5.5│
-    w   ├─▲─┼───┼───┼───┼─▲─┤
-      3 │ │ │   │   │   │ │ │
-        ├─┴─┼───┼───┼───┼─┴─┤
-      4 │ 0 ├───▶1.5◀───┤ 3 │
-        └───┴───┴───┴───┴───┘
-
- -This is why the size of the heightmap has to be \\(2^n + 1\\) -- the \\(+ 1\\) -ensures that the sides are odd lengths, which makes sure they *have* a middle -element for us to set. - -Let's make a couple of helper functions: - - :::clojure - (defn jitter [value spread] - (+ value (rand-around-zero spread))) - - (defn midpoint [a b] - (/ (+ a b) 2)) - - (defn average2 [a b] - (/ (+ a b) 2)) - - (defn average4 [a b c d] - (/ (+ a b c d) 4)) - -`jitter` is just a nice way of saying "take this point and move it a random -amount up or down". - -The other three functions are pretty self-explanatory. Also, two of them are -exactly the same function! There's a reason for this. - -Lately I've started digging into [SICP][] again and watching the [lectures][]. -Abelson and Sussman practice a style of programming that's more about "growing" -a language to talk about a problem you're trying to solve. So you start with -the base programming language, and then define new terms/syntax that let you -talk about your problem at the appropriate level of detail. - -So in this case, we've got two functions for conceptually different tasks: - -* `midpoint` finds the middle point between two points (indexes in an array in - this case). -* `average2` finds the average of two values (floating point height values - here). - -They happen to be implemented in the same way, but I think giving them different -names makes them easier to talk about and easier to keep straight in your head. - - :::clojure - (defn mpd-displace [heightmap lx rx by ty spread] - (let [cx (midpoint lx rx) - cy (midpoint by ty) - - bottom-left (heightmap-get heightmap lx by) - bottom-right (heightmap-get heightmap rx by) - top-left (heightmap-get heightmap lx ty) - top-right (heightmap-get heightmap rx ty) - - top (average2 top-left top-right) - left (average2 bottom-left top-left) - bottom (average2 bottom-left bottom-right) - right (average2 bottom-right top-right)] - (heightmap-set! heightmap cx by (jitter bottom spread)) - (heightmap-set! heightmap cx ty (jitter top spread)) - (heightmap-set! heightmap lx cy (jitter left spread)) - (heightmap-set! heightmap rx cy (jitter right spread)))) - - (defn midpoint-displacement-d2 [heightmap] - (mpd-init-corners heightmap) - (mpd-displace heightmap - 0 heightmap.last - 0 heightmap.last - 0.1)) - -`mpd-displace` is the meat of the algorithm. It takes a heightmap, the spread -(how much random "jitter" to apply after averaging the values), and four values -to define the square we're working on in the heightmap: - -* `lx`: the x coordinate for the left-hand corners -* `rx`: the x coordinate for the right-hand corners -* `by`: the y coordinate for the bottom corners -* `ty`: the y coordinate for the top corners - -For the first iteration we want to work on the entire heightmap, so we pass in -`0` and `heightmap.last` for the left/right and top/bottom indices. - -Run the demo a few times and watch how the midpoints fall between the corner -points because they're averaged (with a little bit of jitter thrown in). - -
- -[SICP]: http://www.amazon.com/dp/0262510871/?tag=stelos-20 -[lectures]: https://youtu.be/2Op3QLzMgSY - -### Set the Center - -The next step is to set the center element of the square to the average of those -midpoints we just set, plus a bit of jitter: - -
-               Column
-          0   1   2   3   4
-        ┌───┬───┬───┬───┬───┐
-      0 │ 2 │   │ 5 │   │ 8 │
-        ├───┼───┼───┼───┼───┤
-      1 │   │   │   │   │   │
-    R   ├───┼───┼───┼───┼───┤
-    o 2 │ 1 │   │ ◉ │   │5.5│
-    w   ├───┼───┼───┼───┼───┤
-      3 │   │   │   │   │   │
-        ├───┼───┼───┼───┼───┤
-      4 │ 0 │   │1.5│   │ 3 │
-        └───┴───┴───┴───┴───┘
-
-
-               Column
-          0   1   2   3   4
-        ┌───┬───┬───┬───┬───┐
-      0 │ 2 │   │ 5 │   │ 8 │
-        ├───┼───┼─┬─┼───┼───┤
-      1 │   │   │ │ │   │   │
-    R   ├───┼───┼─▼─┼───┼───┤
-    o 2 │ 1 ├───▶ ◉ ◀───┤5.5│
-    w   ├───┼───┼─▲─┼───┼───┤
-      3 │   │   │ │ │   │   │
-        ├───┼───┼─┴─┼───┼───┤
-      4 │ 0 │   │1.5│   │ 3 │
-        └───┴───┴───┴───┴───┘
-
-
-               Column
-          0   1   2   3   4
-        ┌───┬───┬───┬───┬───┐
-      0 │ 2 │   │ 5 │   │ 8 │
-        ├───┼───┼─┬─┼───┼───┤
-      1 │   │   │ │ │   │   │
-    R   ├───┼───┼─▼─┼───┼───┤
-    o 2 │ 1 ├───▶3.3◀───┤5.5│
-    w   ├───┼───┼─▲─┼───┼───┤
-      3 │   │   │ │ │   │   │
-        ├───┼───┼─┴─┼───┼───┤
-      4 │ 0 │   │1.5│   │ 3 │
-        └───┴───┴───┴───┴───┘
-
- -We can add this to the displacement function pretty easily: - - :::clojure - (defn mpd-displace [heightmap lx rx by ty spread] - (let [cx (midpoint lx rx) - cy (midpoint by ty) - - bottom-left (heightmap-get heightmap lx by) - bottom-right (heightmap-get heightmap rx by) - top-left (heightmap-get heightmap lx ty) - top-right (heightmap-get heightmap rx ty) - - top (average2 top-left top-right) - left (average2 bottom-left top-left) - bottom (average2 bottom-left bottom-right) - right (average2 bottom-right top-right) - center (average4 top left bottom right)] - (heightmap-set! heightmap cx by (jitter bottom spread)) - (heightmap-set! heightmap cx ty (jitter top spread)) - (heightmap-set! heightmap lx cy (jitter left spread)) - (heightmap-set! heightmap rx cy (jitter right spread)) - (heightmap-set! heightmap cx cy (jitter center spread)))) - -And now our center is ready to go: - -
- -### Iterate - -To finish things off, we need to iterate so that we can fill in the rest of the -values in progressively smaller squares. - -A 3x3 square is the base case, because its corners will be filled in to start, -and the edges and midpoint get filled in. We just call our displacement -function with the top/bottom/left/right values defining the square and we're -done. - -A 5x5 starts with one iteration that displaces the whole map. Then it does -another "pass" that displaces each of the 3x3 "sub-squares": - -
-    ╔═══════════╗───┬───┐      ┌───┬───╔═══════════╗
-    ║ 2 │   │ 5 ║   │ 8 │      │ 2 │   ║ 5 │   │ 8 ║
-    ║───┼───┼───║───┼───┤      ├───┼───║───┼───┼───║
-    ║   │   │   ║   │   │      │   │   ║   │   │   ║
-    ║───┼───┼───║───┼───┤      ├───┼───║───┼───┼───║
-    ║ 1 │   │3.3║   │5.5│      │ 1 │   ║3.3│   │5.5║
-    ╚═══════════╝───┼───┤      ├───┼───╚═══════════╝
-    │   │   │   │   │   │      │   │   │   │   │   │
-    ├───┼───┼───┼───┼───┤      ├───┼───┼───┼───┼───┤
-    │ 0 │   │1.5│   │ 3 │      │ 0 │   │1.5│   │ 3 │
-    └───┴───┴───┴───┴───┘      └───┴───┴───┴───┴───┘
-
-
-    ┌───┬───┬───┬───┬───┐      ┌───┬───┬───┬───┬───┐
-    │ 2 │   │ 5 │   │ 8 │      │ 2 │   │ 5 │   │ 8 │
-    ├───┼───┼───┼───┼───┤      ├───┼───┼───┼───┼───┤
-    │   │   │   │   │   │      │   │   │   │   │   │
-    ╔═══════════╗───┼───┤      ├───┼───╔═══════════╗
-    ║ 1 │   │3.3║   │5.5│      │ 1 │   ║3.3│   │5.5║
-    ║───┼───┼───║───┼───┤      ├───┼───║───┼───┼───║
-    ║   │   │   ║   │   │      │   │   ║   │   │   ║
-    ║───┼───┼───║───┼───┤      ├───┼───║───┼───┼───║
-    ║ 0 │   │1.5║   │ 3 │      │ 0 │   ║1.5│   │ 3 ║
-    ╚═══════════╝───┴───┘      └───┴───╚═══════════╝
-
- -If we have a bigger heightmap (e.g. 9x9) we'll need three passes: - -
-     Pass 1
-    ▩▩▩▩▩▩▩▩▩
-    ▩▩▩▩▩▩▩▩▩
-    ▩▩▩▩▩▩▩▩▩
-    ▩▩▩▩▩▩▩▩▩
-    ▩▩▩▩▩▩▩▩▩
-    ▩▩▩▩▩▩▩▩▩
-    ▩▩▩▩▩▩▩▩▩
-    ▩▩▩▩▩▩▩▩▩
-    ▩▩▩▩▩▩▩▩▩
-
-
-     Pass 2
-    ▩▩▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▩▩  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-    ▩▩▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▩▩  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-    ▩▩▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▩▩  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-    ▩▩▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▩▩  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-    ▩▩▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▩▩  ▩▩▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▩▩
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▩▩▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▩▩
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▩▩▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▩▩
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▩▩▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▩▩
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▩▩▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▩▩
-
-
-     Pass 3
-    ▩▩▩▢▢▢▢▢▢  ▢▢▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▢▢  ▢▢▢▢▢▢▩▩▩
-    ▩▩▩▢▢▢▢▢▢  ▢▢▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▢▢  ▢▢▢▢▢▢▩▩▩
-    ▩▩▩▢▢▢▢▢▢  ▢▢▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▢▢  ▢▢▢▢▢▢▩▩▩
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-    ▩▩▩▢▢▢▢▢▢  ▢▢▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▢▢  ▢▢▢▢▢▢▩▩▩
-    ▩▩▩▢▢▢▢▢▢  ▢▢▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▢▢  ▢▢▢▢▢▢▩▩▩
-    ▩▩▩▢▢▢▢▢▢  ▢▢▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▢▢  ▢▢▢▢▢▢▩▩▩
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-    ▩▩▩▢▢▢▢▢▢  ▢▢▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▢▢  ▢▢▢▢▢▢▩▩▩
-    ▩▩▩▢▢▢▢▢▢  ▢▢▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▢▢  ▢▢▢▢▢▢▩▩▩
-    ▩▩▩▢▢▢▢▢▢  ▢▢▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▢▢  ▢▢▢▢▢▢▩▩▩
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
-    ▩▩▩▢▢▢▢▢▢  ▢▢▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▢▢  ▢▢▢▢▢▢▩▩▩
-    ▩▩▩▢▢▢▢▢▢  ▢▢▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▢▢  ▢▢▢▢▢▢▩▩▩
-    ▩▩▩▢▢▢▢▢▢  ▢▢▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▢▢  ▢▢▢▢▢▢▩▩▩
-
- -Let's think about what's happening here. - -* On pass `0`, we break up the grid into a 1x1 set of chunks and displace them (well, "it"). -* On pass `1`, we break up the grid into a 2x2 set of chunks and displace them. -* On pass `2`, we break up the grid into a 4x4 set of chunks and displace them. - -This 9x9 heightmap was made with an exponent of 3 (\\(2^3 + 1 = 9\\)) and we -need 3 "passes" to finish it. This isn't a coincidence -- it's the reason we -chose our heightmap resolutions to be \\(2^n + 1\\). - -So at iteration \\(i\\), we need to displace a \\(2^i\\)x\\(2^i\\) collection of -squares. Let's go: - - :::clojure - (defn midpoint-displacement [heightmap] - (mpd-init-corners heightmap) - (loop [iter 0 - spread 0.3] - (when (< iter heightmap.exponent) - (let [chunks (Math.pow 2 iter) - chunk-width (/ (- heightmap.resolution 1) chunks)] - (do-nested xchunk ychunk chunks - (let [left-x (* chunk-width xchunk) - right-x (+ left-x chunk-width) - bottom-y (* chunk-width ychunk) - top-y (+ bottom-y chunk-width)] - (mpd-displace heightmap - left-x right-x bottom-y top-y - spread)))) - (recur (+ 1 iter) (* spread 0.5)))) - (normalize heightmap)) - -That's a lot to take in. Let's break it down: - - :::clojure - (defn midpoint-displacement [heightmap] - (mpd-init-corners heightmap) - (loop [iter 0 - spread 0.3] - (when (< iter heightmap.exponent) - ; Process the chunks at this iteration - (recur (+ 1 iter) (* spread 0.5)))) - (normalize heightmap)) - -We initialize the corners at the beginning and normalize the heightmap at the -end. Then we loop a number of times equal to the exponent we used to make the -heightmap, as described above. Each time through the loop we increment `iter` -and cut the `spread` for those points in half. - -For each pass, we figure out what we're going to need to do: - - :::clojure - (let [chunks (Math.pow 2 iter) - chunk-width (/ (- heightmap.resolution 1) chunks)] - ; Actually do it... - ) - -Let's make a quick "nested for loop" syntax. It'll make things easier to read -in the main function and we'll need it for later algorithms too: - - :::clojure - (defmacro do-nested [xname yname width & body] - (let [iterations (gensym)] - `(let [~iterations ~width] - (do-times ~xname ~iterations - (do-times ~yname ~iterations - ~@body))))) - -This lets us say something like `(do-nested x y 5 (console.log [x y]))` to mean: - - :::javascript - for (x = 0; x < 5; x++) { - for (y = 0; y < 5; y++) { - console.log([x, y]); - } - } - -And now we can see how we process the chunks: - - :::clojure - (let [chunks (Math.pow 2 iter) - chunk-width (/ (- heightmap.resolution 1) chunks)] - (do-nested xchunk ychunk chunks - (let [left-x (* chunk-width xchunk) - right-x (+ left-x chunk-width) - bottom-y (* chunk-width ychunk) - top-y (+ bottom-y chunk-width)] - (mpd-displace heightmap - left-x right-x bottom-y top-y - spread)))) - -We loop over the indices of the chunks, calculate the bounds for each one, and -displace it. Simple enough, but a bit fiddly to get right. And now we've -finished displacing all the points! - -We could have done this recursively, but I was in an iterative mood when I wrote -this. Feel free to play around with it. - -
- -## Result - -Now that we've got a working algorithm we can play around with the values and -see what effect they have on the resulting terrain. - -**Warning**: don't set the exponent higher than 8 unless you want to crash this -browser tab. An exponent of `8` results in a heightmap with `66049` elements. -Going to `9` is `263169`, which most browsers seem to... dislike. - -
-
-

- -
- -
- -

-
- -And that's it, we've grown some mountains! - -Now that we've got some of the boilerplate down I'm planning on doing a couple -more posts like this looking at other noise algorithms, and maybe some erosion -algorithms to simulate weathering. Let me know if you've got any specific -requests! - - {% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2016/02/midpoint-displacement.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2016/02/midpoint-displacement.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,882 @@ ++++ +title = "Terrain Generation with Midpoint Displacement" +snip = "A first step toward growing worlds with computers." +date = 2016-02-19T19:45:00Z +draft = false + ++++ + + + + + +I'm taking the Game Engine Architecture class at Reykjavík University. My group +just finished our midterm [project][] where we played around with procedural +terrain generation in [Unity][]. It was a lot of fun and I really enjoyed +creating terrain out of numbers, so I thought I'd write up a little introduction +to a few of the algorithms. + +In this post we'll be looking at Midpoint Displacement. + +The full series of posts so far: + +* [Midpoint Displacement](/blog/2016/02/midpoint-displacement/) +* [Recursive Midpoint Displacement](/blog/2016/03/recursive-midpoint-displacement/) +* [Diamond Square](/blog/2016/06/diamond-square/) + +[project]: https://www.youtube.com/watch?v=G5u79w4qiAA +[Unity]: http://unity3d.com/ + +{{% toc %}} + +## Terrain Generation + +Procedural generation is a pretty big field. The [Wikipedia +article][wikipedia-pg] gives a pretty brief overview. The [Procedural Content +Generation Wiki][pcg-wiki] is an entire wiki devoted to procedural generation. +There's also a small but active [subreddit][r-pg] for interested folks. + +Today we're just going to look at a single algorithm for generating +realistic-looking terrain called "Midpoint Displacement". It's relatively +simple (compared to other algorithms) but it produces terrain that actually +looks kind of cool. + +[wikipedia-pg]: https://en.wikipedia.org/wiki/Procedural_generation +[pcg-wiki]: http://pcg.wikidot.com/ +[r-pg]: https://www.reddit.com/r/proceduralgeneration/ + +## Resources, Code, and Examples + +There are a lot of resources for learning how terrain generation algorithms +work. For midpoint displacement I used the [Wikipedia article][wikipedia-ds], +the [PCG Wiki article][pcgw-midpoint], and academic papers like [this +one][paper]. + +Unfortunately, while there are a *lot* of places that describe algorithms like +this, there are relatively few that explain it *thoroughly*. Academic papers in +particular tend to have a really bad case of +[draw-the-rest-of-the-fucking-owl-syndrome][owl]. They tend to slap an equation +or two on the page and move on without showing any code. + +As anyone who's done much programming knows, there's a big difference between +saying something like "set the midpoints of the diamonds to the average of the +corners" and making a computer actually *do* that. How do we iterate over the +array (row-major or column-major)? Does it matter (yes: if you care about cache +coherency)? What about the edges where there are fewer corners? How random +(and how fast) is our random number generator (and how much do we care)? How do +we write all this code so we can actually read it six months later? + +So I'm going to try to do my part to improve the situation by not just +describing the algorithm, but showing *actual code* that runs and generates +a terrain. + +The full code is [here][code], but we'll be going through the important parts +below. + +[wikipedia-ds]: https://en.wikipedia.org/wiki/Diamond-square_algorithm +[pcgw-midpoint]: http://pcg.wikidot.com/pcg-algorithm:midpoint-displacement-algorithm +[paper]: http://micsymposium.org/mics_2011_proceedings/mics2011_submission_30.pdf +[owl]: https://i.imgur.com/RadSf.jpg +[code]: https://bitbucket.org/sjl/stevelosh/src/default/media/js/terrain1.wisp + +### Wisp + +I'm using a language called [Wisp][wisp]. It's a small, Clojure-inspired +dialect of Lisp that compiles down to vanilla Javascript. + +I went with Javascript because it means I can actually put demos inline in this +post. Sometimes it's so much easier to understand something when you can +actually see it running. + +I've tried to write the code so that it's readable even if you don't know Lisp. +The point of me showing you this code is not to give you something to copy and +paste. The point is that until you *actually write and run* the code, you don't +know all the edge cases and problems that can happen. So by making examples +that actually run I feel a bit more confident that I'm explaining things enough. + +I haven't focused on performance at all. If you want a fast implementation of +this, you'll probably want to use a language with real arrays (i.e. contiguous +chunks of memory holding a bunch of unboxed floats or integers). + +[wisp]: https://github.com/Gozala/wisp + +### Three.js + +[Three.js][three.js] is a Javascript library for 3D rendering. I'm using it +here to get something on the screen you can play with. I'm not going to cover +the code to put the terrains on the screen because it's really an entirely +separate problem to *generating* the terrain, and it's mostly uninteresting +boilerplate for these little demos. + +I haven't been able to get these demos working on my iPhone. Sorry about that, +but I'm not much of a frontend developer. You'll have to use a computer to see +them. [Pull requests][slc] are welcome. + +[slc]: http://bitbucket.org/sjl/stevelosh/ +[three.js]: http://threejs.org/ + +## Heightmaps + +Let's get started. The main data structure we're going to be using is +a [heightmap][]. + +### Overview + +Heightmaps are essentially just big two-dimensional arrays of numbers in a given +range that represent the height of the ground at a given point. For example, +a heightmap with a small "mountain" in the middle might look like: + +```text +[[0, 0, 2, 0, 0], + [0, 2, 5, 1, 0], + [2, 5, 9, 4, 1], + [0, 1, 4, 1, 0], + [0, 0, 1, 0, 0]] +``` + +In practice, heightmaps are often represented as a single-dimensional array +instead to ensure that all the data is together in one big hunk of memory: + +```text +[0, 0, 2, 0, 0, + 0, 2, 5, 1, 0, + 2, 5, 9, 4, 1, + 0, 1, 4, 1, 0, + 0, 0, 1, 0, 0] +``` + +The type and range of the numbers in the map varies by program and programmer. +Some programs use nonzero integers, some use all the integers, some use floating +point, etc. + +The heightmaps in this post will contain floats from `0.0` to `1.0`. I like +using floats between zero and one because it's easy to roughly visualize them in +your head (e.g. `0.1` is ten percent of the maximum height). + +One common way to visualize heightmaps is by turning them into greyscale images +with one pixel per number, with black pixels for lowest value in the map's range +and white for the highest. The [Wikipedia article][heightmap] has an example of +this. This is handy because you can also write some code to *read* images, and +then you can use any image editor to draw your own heightmaps by hand. + +We're going to visualize the heightmaps in 3D. It's a lot more fun, and they +start to actually look like real terrains. + +[heightmap]: https://en.wikipedia.org/wiki/Heightmap + +### Code + +We're going to start by writing a few functions that will hide away the internal +details of how our heightmaps are implemented, so we can talk about them in +nice high-level terms for the rest of the program. We'll also need a couple of +helper functions along the way. + +We'll start with something to create a heightmap: + +```clojure +(defn make-heightmap [exponent] + (let [resolution (+ 1 (Math.pow 2 exponent))] + (l (+ "Creating " resolution " by " resolution " heightmap...")) + (def heightmap + (new Array (* resolution resolution))) + (set! heightmap.resolution resolution) + (set! heightmap.exponent exponent) + (set! heightmap.last (- resolution 1)) + (zero-heightmap heightmap))) +``` + +`make-heightmap` takes an exponent and creates a heightmap. For reasons that +will become clear later, all of our heightmaps must be square, and they'll need +to have \\(2^n + 1\\) rows and columns. This means we can make heightmaps of +sizes like 3x3, 5x5, 9x9, 17x17, 33x33, etc. So our function just takes the +\\(n\\) to use and creates a correctly-sized array. + +(If you've ever poked around with Unity's Terrain objects you might recognize +these "powers of two plus one".) + +We create a new array (1-dimensional) and store a few pieces of extra data on it +for later use. `last` is the index of the last row/column, so for a 3x3 array +it would be `2`. + +`l` is just a simple little logging function: + +```clojure +(defn l [v] + (console.log v)) +``` + +We'll need to zero out the heightmap as well: + +```clojure +(defn zero-heightmap [heightmap] + (do-times i heightmap.length + (set! (aget heightmap i) 0.0)) + heightmap) +``` + +`zero-heightmap` just iterates from `0` to `heightmap.length` and sets each +element to `0.0`. Wisp doesn't have syntax to loop from 0 to some number, but +it's a Lisp, so we can make some: + +```clojure +(defmacro when [condition & body] + `(if ~condition + (do ~@body))) + +(defmacro do-times [varname limit & body] + (let [end (gensym)] + `(let [~end ~limit] + (loop [~varname 0] + (when (< ~varname ~end) + ~@body + (recur (+ 1 ~varname))))))) +``` + +If you're not a Lisp person, don't sweat this too much. Just trust me that +`(do-times i 10 (console.log i))` will print the numbers `0` to `9`. + +Now that we can create a heightmap, we'll probably want to be able to read its +elements: + +```clojure +(defmacro heightmap-get [hm x y] + `(aget ~hm (+ (* ~y (.-resolution ~hm)) ~x))) + +(defn heightmap-get-safe [hm x y] + (when (and (<= 0 x hm.last) + (<= 0 y hm.last)) + (heightmap-get hm x y))) +``` + +`heightmap-get` handles the nasty business of translating our human-thinkable +`x` and `y` coordinates into an index into the single-dimensional array. It's +not particularly fast (especially if the compiler won't inline it), but we're +not worried about speed for this post. + +`heightmap-get-safe` is a version of `heightmap-get` that will do bounds +checking for us and return `nil` if we ask for something out of range. +Javascript will return `undefined` if you ask for a non-existent index, but +because we're storing the array as one big line of data, using a `y` value +that's too big will actually "wrap around" to the next row, so it's safer to +just check it explicitly. + +Of course we'll also want to be able to set values in our heightmap: + +```clojure +(defmacro heightmap-set! [hm x y val] + `(set! (heightmap-get ~hm ~x ~y) ~val)) + +``` + +And finally, we'll want to be able to "normalize" our array. During the course +of running our algorithm, it's possible for the values to become less than `0.0` +or greater than `1.0`. We want our heightmaps to contain only values between +zero and one, so rather than change the algorithm we can just loop through the +array at the end and "squeeze" the range down to 0 and 1 (or "stretch" it if +it ends up smaller): + +```clojure +(defn normalize [hm] + (let [max (- Infinity) + min Infinity] + (do-times i hm.length + (let [el (aget hm i)] + (when (< max el) (set! max el)) + (when (> min el) (set! min el)))) + (let [span (- max min)] + (do-times i hm.length + (set! (aget hm i) + (/ (- (aget hm i) min) + span)))))) +``` + +## Random Noise + +Now that we've got a nice little interface to heightmaps we can move on to +actually making some terrain! Before we dive into midpoint displacement, let's +get a [black triangle][] on the screen. We'll start by just setting every point +in the heightmap to a completely random value. + +```clojure +(defn rand [] + (Math.random)) + +(defn rand-around-zero [spread] + (- (* spread (rand) 2) spread)) + +(defn random-noise [heightmap] + (do-times i heightmap.length + (set! (aget heightmap i) (rand)))) +``` + +Let's see it! Hit the "refresh" button to create and render the terrain. You +can keep hitting it to get fresh terrains. + +
+ +Not really a very fun landscape to explore, but at least we've got something +running. Onward to the real algorithm. + +[black triangle]: http://rampantgames.com/blog/?p=7745 + +## Midpoint Displacement + +Midpoint displacement is a simple, fast way to generate decent-looking terrain. +It's not perfect (you'll notice patterns once you start viewing the mesh from +different angles) but it's a good place to start. + +The general process goes like this: + +1. Initialize the four corners of the heightmap to random values. +2. Set the midpoints of each edge to the average of the two corners it's + between, plus or minus a random amount. +3. Set the center of the square to the average of those edge midpoints you just + set, again with a random jitter. +4. Recurse on the four squares inside this one, reducing the jitter. + +That's the standard description. Now let's draw the fucking owl. + +### Initialize the Corners + +The first step is pretty easy: we just need to take our heightmap: + +
+               Column
+          0   1   2   3   4
+        ┌───┬───┬───┬───┬───┐
+      0 │   │   │   │   │   │
+        ├───┼───┼───┼───┼───┤
+      1 │   │   │   │   │   │
+    R   ├───┼───┼───┼───┼───┤
+    o 2 │   │   │   │   │   │
+    w   ├───┼───┼───┼───┼───┤
+      3 │   │   │   │   │   │
+        ├───┼───┼───┼───┼───┤
+      4 │   │   │   │   │   │
+        └───┴───┴───┴───┴───┘
+
+ +Find the corners: + +
+               Column
+          0   1   2   3   4
+        ┌───┬───┬───┬───┬───┐
+      0 │ ◉ │   │   │   │ ◉ │
+        ├───┼───┼───┼───┼───┤
+      1 │   │   │   │   │   │
+    R   ├───┼───┼───┼───┼───┤
+    o 2 │   │   │   │   │   │
+    w   ├───┼───┼───┼───┼───┤
+      3 │   │   │   │   │   │
+        ├───┼───┼───┼───┼───┤
+      4 │ ◉ │   │   │   │ ◉ │
+        └───┴───┴───┴───┴───┘
+
+ +And shove random values in them: + +
+               Column
+          0   1   2   3   4
+        ┌───┬───┬───┬───┬───┐
+      0 │ 2 │   │   │   │ 8 │
+        ├───┼───┼───┼───┼───┤
+      1 │   │   │   │   │   │
+    R   ├───┼───┼───┼───┼───┤
+    o 2 │   │   │   │   │   │
+    w   ├───┼───┼───┼───┼───┤
+      3 │   │   │   │   │   │
+        ├───┼───┼───┼───┼───┤
+      4 │ 0 │   │   │   │ 3 │
+        └───┴───┴───┴───┴───┘
+
+ +(I'm using 0 to 10 instead of 0 to 1 because it's easier to read in the +ASCII-art diagrams.) + +The code is about what you'd expect: + +```clojure +(defn mpd-init-corners [heightmap] + (heightmap-set! heightmap 0 0 (rand)) + (heightmap-set! heightmap 0 heightmap.last (rand)) + (heightmap-set! heightmap heightmap.last 0 (rand)) + (heightmap-set! heightmap heightmap.last heightmap.last (rand))) + +(defn midpoint-displacement-d1 [heightmap] + (mpd-init-corners heightmap)) +``` + +That was easy. + +
+ +### Set the Edges + +For the next step, we need to take our square heightmap with filled-in corners +and use the corners to fill in the middle element of each edge: + +
+               Column
+          0   1   2   3   4
+        ┌───┬───┬───┬───┬───┐
+      0 │ 2 │   │ ◉ │   │ 8 │
+        ├───┼───┼───┼───┼───┤
+      1 │   │   │   │   │   │
+    R   ├───┼───┼───┼───┼───┤
+    o 2 │ ◉ │   │   │   │ ◉ │
+    w   ├───┼───┼───┼───┼───┤
+      3 │   │   │   │   │   │
+        ├───┼───┼───┼───┼───┤
+      4 │ 0 │   │ ◉ │   │ 3 │
+        └───┴───┴───┴───┴───┘
+
+
+
+               Column
+          0   1   2   3   4
+        ┌───┬───┬───┬───┬───┐
+      0 │ 2 ├───▶ ◉ ◀───┤ 8 │
+        ├─┬─┼───┼───┼───┼─┬─┤
+      1 │ │ │   │   │   │ │ │
+    R   ├─▼─┼───┼───┼───┼─▼─┤
+    o 2 │ ◉ │   │   │   │ ◉ │
+    w   ├─▲─┼───┼───┼───┼─▲─┤
+      3 │ │ │   │   │   │ │ │
+        ├─┴─┼───┼───┼───┼─┴─┤
+      4 │ 0 ├───▶ ◉ ◀───┤ 3 │
+        └───┴───┴───┴───┴───┘
+
+
+               Column
+          0   1   2   3   4
+        ┌───┬───┬───┬───┬───┐
+      0 │ 2 ├───▶ 5 ◀───┤ 8 │
+        ├─┬─┼───┼───┼───┼─┬─┤
+      1 │ │ │   │   │   │ │ │
+    R   ├─▼─┼───┼───┼───┼─▼─┤
+    o 2 │ 1 │   │   │   │5.5│
+    w   ├─▲─┼───┼───┼───┼─▲─┤
+      3 │ │ │   │   │   │ │ │
+        ├─┴─┼───┼───┼───┼─┴─┤
+      4 │ 0 ├───▶1.5◀───┤ 3 │
+        └───┴───┴───┴───┴───┘
+
+ +This is why the size of the heightmap has to be \\(2^n + 1\\) — the \\(+ 1\\) +ensures that the sides are odd lengths, which makes sure they *have* a middle +element for us to set. + +Let's make a couple of helper functions: + +```clojure +(defn jitter [value spread] + (+ value (rand-around-zero spread))) + +(defn midpoint [a b] + (/ (+ a b) 2)) + +(defn average2 [a b] + (/ (+ a b) 2)) + +(defn average4 [a b c d] + (/ (+ a b c d) 4)) +``` + +`jitter` is just a nice way of saying "take this point and move it a random +amount up or down". + +The other three functions are pretty self-explanatory. Also, two of them are +exactly the same function! There's a reason for this. + +Lately I've started digging into [SICP][] again and watching the [lectures][]. +Abelson and Sussman practice a style of programming that's more about "growing" +a language to talk about a problem you're trying to solve. So you start with +the base programming language, and then define new terms/syntax that let you +talk about your problem at the appropriate level of detail. + +So in this case, we've got two functions for conceptually different tasks: + +* `midpoint` finds the middle point between two points (indexes in an array in + this case). +* `average2` finds the average of two values (floating point height values + here). + +They happen to be implemented in the same way, but I think giving them different +names makes them easier to talk about and easier to keep straight in your head. + +```clojure +(defn mpd-displace [heightmap lx rx by ty spread] + (let [cx (midpoint lx rx) + cy (midpoint by ty) + + bottom-left (heightmap-get heightmap lx by) + bottom-right (heightmap-get heightmap rx by) + top-left (heightmap-get heightmap lx ty) + top-right (heightmap-get heightmap rx ty) + + top (average2 top-left top-right) + left (average2 bottom-left top-left) + bottom (average2 bottom-left bottom-right) + right (average2 bottom-right top-right)] + (heightmap-set! heightmap cx by (jitter bottom spread)) + (heightmap-set! heightmap cx ty (jitter top spread)) + (heightmap-set! heightmap lx cy (jitter left spread)) + (heightmap-set! heightmap rx cy (jitter right spread)))) + +(defn midpoint-displacement-d2 [heightmap] + (mpd-init-corners heightmap) + (mpd-displace heightmap + 0 heightmap.last + 0 heightmap.last + 0.1)) +``` + +`mpd-displace` is the meat of the algorithm. It takes a heightmap, the spread +(how much random "jitter" to apply after averaging the values), and four values +to define the square we're working on in the heightmap: + +* `lx`: the x coordinate for the left-hand corners +* `rx`: the x coordinate for the right-hand corners +* `by`: the y coordinate for the bottom corners +* `ty`: the y coordinate for the top corners + +For the first iteration we want to work on the entire heightmap, so we pass in +`0` and `heightmap.last` for the left/right and top/bottom indices. + +Run the demo a few times and watch how the midpoints fall between the corner +points because they're averaged (with a little bit of jitter thrown in). + +
+ +[SICP]: http://www.amazon.com/dp/0262510871/?tag=stelos-20 +[lectures]: https://youtu.be/2Op3QLzMgSY + +### Set the Center + +The next step is to set the center element of the square to the average of those +midpoints we just set, plus a bit of jitter: + +
+               Column
+          0   1   2   3   4
+        ┌───┬───┬───┬───┬───┐
+      0 │ 2 │   │ 5 │   │ 8 │
+        ├───┼───┼───┼───┼───┤
+      1 │   │   │   │   │   │
+    R   ├───┼───┼───┼───┼───┤
+    o 2 │ 1 │   │ ◉ │   │5.5│
+    w   ├───┼───┼───┼───┼───┤
+      3 │   │   │   │   │   │
+        ├───┼───┼───┼───┼───┤
+      4 │ 0 │   │1.5│   │ 3 │
+        └───┴───┴───┴───┴───┘
+
+
+               Column
+          0   1   2   3   4
+        ┌───┬───┬───┬───┬───┐
+      0 │ 2 │   │ 5 │   │ 8 │
+        ├───┼───┼─┬─┼───┼───┤
+      1 │   │   │ │ │   │   │
+    R   ├───┼───┼─▼─┼───┼───┤
+    o 2 │ 1 ├───▶ ◉ ◀───┤5.5│
+    w   ├───┼───┼─▲─┼───┼───┤
+      3 │   │   │ │ │   │   │
+        ├───┼───┼─┴─┼───┼───┤
+      4 │ 0 │   │1.5│   │ 3 │
+        └───┴───┴───┴───┴───┘
+
+
+               Column
+          0   1   2   3   4
+        ┌───┬───┬───┬───┬───┐
+      0 │ 2 │   │ 5 │   │ 8 │
+        ├───┼───┼─┬─┼───┼───┤
+      1 │   │   │ │ │   │   │
+    R   ├───┼───┼─▼─┼───┼───┤
+    o 2 │ 1 ├───▶3.3◀───┤5.5│
+    w   ├───┼───┼─▲─┼───┼───┤
+      3 │   │   │ │ │   │   │
+        ├───┼───┼─┴─┼───┼───┤
+      4 │ 0 │   │1.5│   │ 3 │
+        └───┴───┴───┴───┴───┘
+
+ +We can add this to the displacement function pretty easily: + +```clojure +(defn mpd-displace [heightmap lx rx by ty spread] + (let [cx (midpoint lx rx) + cy (midpoint by ty) + + bottom-left (heightmap-get heightmap lx by) + bottom-right (heightmap-get heightmap rx by) + top-left (heightmap-get heightmap lx ty) + top-right (heightmap-get heightmap rx ty) + + top (average2 top-left top-right) + left (average2 bottom-left top-left) + bottom (average2 bottom-left bottom-right) + right (average2 bottom-right top-right) + center (average4 top left bottom right)] + (heightmap-set! heightmap cx by (jitter bottom spread)) + (heightmap-set! heightmap cx ty (jitter top spread)) + (heightmap-set! heightmap lx cy (jitter left spread)) + (heightmap-set! heightmap rx cy (jitter right spread)) + (heightmap-set! heightmap cx cy (jitter center spread)))) +``` + +And now our center is ready to go: + +
+ +### Iterate + +To finish things off, we need to iterate so that we can fill in the rest of the +values in progressively smaller squares. + +A 3x3 square is the base case, because its corners will be filled in to start, +and the edges and midpoint get filled in. We just call our displacement +function with the top/bottom/left/right values defining the square and we're +done. + +A 5x5 starts with one iteration that displaces the whole map. Then it does +another "pass" that displaces each of the 3x3 "sub-squares": + +
+    ╔═══════════╗───┬───┐      ┌───┬───╔═══════════╗
+    ║ 2 │   │ 5 ║   │ 8 │      │ 2 │   ║ 5 │   │ 8 ║
+    ║───┼───┼───║───┼───┤      ├───┼───║───┼───┼───║
+    ║   │   │   ║   │   │      │   │   ║   │   │   ║
+    ║───┼───┼───║───┼───┤      ├───┼───║───┼───┼───║
+    ║ 1 │   │3.3║   │5.5│      │ 1 │   ║3.3│   │5.5║
+    ╚═══════════╝───┼───┤      ├───┼───╚═══════════╝
+    │   │   │   │   │   │      │   │   │   │   │   │
+    ├───┼───┼───┼───┼───┤      ├───┼───┼───┼───┼───┤
+    │ 0 │   │1.5│   │ 3 │      │ 0 │   │1.5│   │ 3 │
+    └───┴───┴───┴───┴───┘      └───┴───┴───┴───┴───┘
+
+
+    ┌───┬───┬───┬───┬───┐      ┌───┬───┬───┬───┬───┐
+    │ 2 │   │ 5 │   │ 8 │      │ 2 │   │ 5 │   │ 8 │
+    ├───┼───┼───┼───┼───┤      ├───┼───┼───┼───┼───┤
+    │   │   │   │   │   │      │   │   │   │   │   │
+    ╔═══════════╗───┼───┤      ├───┼───╔═══════════╗
+    ║ 1 │   │3.3║   │5.5│      │ 1 │   ║3.3│   │5.5║
+    ║───┼───┼───║───┼───┤      ├───┼───║───┼───┼───║
+    ║   │   │   ║   │   │      │   │   ║   │   │   ║
+    ║───┼───┼───║───┼───┤      ├───┼───║───┼───┼───║
+    ║ 0 │   │1.5║   │ 3 │      │ 0 │   ║1.5│   │ 3 ║
+    ╚═══════════╝───┴───┘      └───┴───╚═══════════╝
+
+ +If we have a bigger heightmap (e.g. 9x9) we'll need three passes: + +
+     Pass 1
+    ▩▩▩▩▩▩▩▩▩
+    ▩▩▩▩▩▩▩▩▩
+    ▩▩▩▩▩▩▩▩▩
+    ▩▩▩▩▩▩▩▩▩
+    ▩▩▩▩▩▩▩▩▩
+    ▩▩▩▩▩▩▩▩▩
+    ▩▩▩▩▩▩▩▩▩
+    ▩▩▩▩▩▩▩▩▩
+    ▩▩▩▩▩▩▩▩▩
+
+
+     Pass 2
+    ▩▩▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▩▩  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+    ▩▩▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▩▩  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+    ▩▩▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▩▩  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+    ▩▩▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▩▩  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+    ▩▩▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▩▩  ▩▩▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▩▩
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▩▩▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▩▩
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▩▩▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▩▩
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▩▩▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▩▩
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▩▩▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▩▩
+
+
+     Pass 3
+    ▩▩▩▢▢▢▢▢▢  ▢▢▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▢▢  ▢▢▢▢▢▢▩▩▩
+    ▩▩▩▢▢▢▢▢▢  ▢▢▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▢▢  ▢▢▢▢▢▢▩▩▩
+    ▩▩▩▢▢▢▢▢▢  ▢▢▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▢▢  ▢▢▢▢▢▢▩▩▩
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+    ▩▩▩▢▢▢▢▢▢  ▢▢▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▢▢  ▢▢▢▢▢▢▩▩▩
+    ▩▩▩▢▢▢▢▢▢  ▢▢▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▢▢  ▢▢▢▢▢▢▩▩▩
+    ▩▩▩▢▢▢▢▢▢  ▢▢▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▢▢  ▢▢▢▢▢▢▩▩▩
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+    ▩▩▩▢▢▢▢▢▢  ▢▢▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▢▢  ▢▢▢▢▢▢▩▩▩
+    ▩▩▩▢▢▢▢▢▢  ▢▢▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▢▢  ▢▢▢▢▢▢▩▩▩
+    ▩▩▩▢▢▢▢▢▢  ▢▢▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▢▢  ▢▢▢▢▢▢▩▩▩
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+    ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢  ▢▢▢▢▢▢▢▢▢
+    ▩▩▩▢▢▢▢▢▢  ▢▢▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▢▢  ▢▢▢▢▢▢▩▩▩
+    ▩▩▩▢▢▢▢▢▢  ▢▢▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▢▢  ▢▢▢▢▢▢▩▩▩
+    ▩▩▩▢▢▢▢▢▢  ▢▢▩▩▩▢▢▢▢  ▢▢▢▢▩▩▩▢▢  ▢▢▢▢▢▢▩▩▩
+
+ +Let's think about what's happening here. + +* On pass `0`, we break up the grid into a 1x1 set of chunks and displace them (well, "it"). +* On pass `1`, we break up the grid into a 2x2 set of chunks and displace them. +* On pass `2`, we break up the grid into a 4x4 set of chunks and displace them. + +This 9x9 heightmap was made with an exponent of 3 (\\(2^3 + 1 = 9\\)) and we +need 3 "passes" to finish it. This isn't a coincidence — it's the reason we +chose our heightmap resolutions to be \\(2^n + 1\\). + +So at iteration \\(i\\), we need to displace a \\(2^i\\)x\\(2^i\\) collection of +squares. Let's go: + +```clojure +(defn midpoint-displacement [heightmap] + (mpd-init-corners heightmap) + (loop [iter 0 + spread 0.3] + (when (< iter heightmap.exponent) + (let [chunks (Math.pow 2 iter) + chunk-width (/ (- heightmap.resolution 1) chunks)] + (do-nested xchunk ychunk chunks + (let [left-x (* chunk-width xchunk) + right-x (+ left-x chunk-width) + bottom-y (* chunk-width ychunk) + top-y (+ bottom-y chunk-width)] + (mpd-displace heightmap + left-x right-x bottom-y top-y + spread)))) + (recur (+ 1 iter) (* spread 0.5)))) + (normalize heightmap)) +``` + +That's a lot to take in. Let's break it down: + +```clojure +(defn midpoint-displacement [heightmap] + (mpd-init-corners heightmap) + (loop [iter 0 + spread 0.3] + (when (< iter heightmap.exponent) + ; Process the chunks at this iteration + (recur (+ 1 iter) (* spread 0.5)))) + (normalize heightmap)) +``` + +We initialize the corners at the beginning and normalize the heightmap at the +end. Then we loop a number of times equal to the exponent we used to make the +heightmap, as described above. Each time through the loop we increment `iter` +and cut the `spread` for those points in half. + +For each pass, we figure out what we're going to need to do: + +```clojure +(let [chunks (Math.pow 2 iter) + chunk-width (/ (- heightmap.resolution 1) chunks)] + ; Actually do it... + ) +``` + +Let's make a quick "nested for loop" syntax. It'll make things easier to read +in the main function and we'll need it for later algorithms too: + +```clojure +(defmacro do-nested [xname yname width & body] + (let [iterations (gensym)] + `(let [~iterations ~width] + (do-times ~xname ~iterations + (do-times ~yname ~iterations + ~@body))))) +``` + +This lets us say something like `(do-nested x y 5 (console.log [x y]))` to mean: + +```javascript +for (x = 0; x < 5; x++) { + for (y = 0; y < 5; y++) { + console.log([x, y]); + } +} +``` + +And now we can see how we process the chunks: + +```clojure +(let [chunks (Math.pow 2 iter) + chunk-width (/ (- heightmap.resolution 1) chunks)] + (do-nested xchunk ychunk chunks + (let [left-x (* chunk-width xchunk) + right-x (+ left-x chunk-width) + bottom-y (* chunk-width ychunk) + top-y (+ bottom-y chunk-width)] + (mpd-displace heightmap + left-x right-x bottom-y top-y + spread)))) +``` + +We loop over the indices of the chunks, calculate the bounds for each one, and +displace it. Simple enough, but a bit fiddly to get right. And now we've +finished displacing all the points! + +We could have done this recursively, but I was in an iterative mood when I wrote +this. Feel free to play around with it. + +
+ +## Result + +Now that we've got a working algorithm we can play around with the values and +see what effect they have on the resulting terrain. + +**Warning**: don't set the exponent higher than 8 unless you want to crash this +browser tab. An exponent of `8` results in a heightmap with `66049` elements. +Going to `9` is `263169`, which most browsers seem to... dislike. + +
+
+

+ +
+ +
+ +

+
+ +And that's it, we've grown some mountains! + +Now that we've got some of the boilerplate down I'm planning on doing a couple +more posts like this looking at other noise algorithms, and maybe some erosion +algorithms to simulate weathering. Let me know if you've got any specific +requests! + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2016/03/recursive-midpoint-displacement.html --- a/content/blog/2016/03/recursive-midpoint-displacement.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,274 +0,0 @@ - {% extends "_post.html" %} - - {% load mathjax %} - - {% hyde - title: "Recursive Midpoint Displacement" - snip: "A cleaner version." - created: 2016-03-07 13:45:00 - %} - - {% block extra_js %} - - - - - {% endblock extra_js %} - - {% block article %} - -In the [last post][mpd] we looked at implementing the Midpoint Displacement -algorithm. I ended up doing the last step iteratively, which works, but isn't -the cleanest way to do it. Before moving on to other algorithms I wanted to -clean things up by using a handy library. - -The full series of posts so far: - -* [Midpoint Displacement](/blog/2016/02/midpoint-displacement/) -* [Recursive Midpoint Displacement](/blog/2016/03/recursive-midpoint-displacement/) -* [Diamond Square](/blog/2016/06/diamond-square/) - -[mpd]: /blog/2016/02/midpoint-displacement/ - -[TOC] - -## Multi-Dimensional Arrays - -Last week when looking at something unrelated I came across the [ndarray][] -library ("nd" stands for "n-dimensional"). This is a little wrapper around -standard Javascript arrays to add easy multi-dimensional indexing. We're going -to to use `Float64Array` objects as the underlying storage because they're much -more efficient than the vanilla Javascript arrays and they're fairly well -supported. - -It also adds [slicing][], which is a lot like Common Lisp's [displaced -arrays][disp-arr]. It lets you create a new "array" object with a different -"shape" that doesn't have any actual storage of its own, but instead refers back -to the original array's data. This is perfect for implementing Midpoint -Displacement with recursion. - -[ndarray]: https://github.com/scijs/ndarray -[slicing]: https://github.com/scijs/ndarray#slicing -[disp-arr]: http://clhs.lisp.se/Body/26_glo_d.htm#displaced_array - -## Iteration - -We're still going to need to iterate over ndarrays at certain points (e.g. when -normalizing them) so let's make some helpful macros to do the annoying busywork -for us: - - :::clojure - (defmacro do-ndarray [vars array-form & body] - (let [array-var (gensym "array") - build - (fn build [vars n] - (if (empty? vars) - `(do ~@body) - `(do-times ~(first vars) (aget (.-shape ~array-var) ~n) - ~(build (rest vars) (inc n)))))] - `(let [~array-var ~array-form] - ~(build vars 0)))) - - (defmacro do-ndarray-el [element array-form & body] - (let [index (gensym "index") - array (gensym "array")] - `(let [~array ~array-form] - (do-times ~index (.-length (.-data ~array)) - (let [~element (aget (.-data ~array) ~index)] - ~@body))))) - -Now we can easily iterate over the indices: - - :::clojure - (do-ndarray [x y] my-ndarray - (console.log "Array[" x "][" y "] is: " - (.get my-ndarray x y))) - -Or just over the items if we don't need their indices: - - :::clojure - (do-ndarray-el item my-ndarray - (console.log item)) - -These macros should work for ndarrays of any number of dimensions, and will -compile into ugly but fast Javascript `for` loops. - -## Updating the Heightmaps - -To start we'll need to update the heightmap functions to work with ndarrays -instead of the normal JS arrays. - -Start with a few functions to calculate sizes/incides: - - :::clojure - (defn heightmap-resolution [heightmap] - (aget heightmap.shape 0)) - - (defn heightmap-last-index [heightmap] - (dec (heightmap-resolution heightmap))) - - (defn heightmap-center-index [heightmap] - (midpoint 0 (heightmap-last-index heightmap))) - -Support for reading/writing: - - :::clojure - (defn heightmap-get [heightmap x y] - (.get heightmap x y)) - - (defn heightmap-get-safe [heightmap x y] - (let [last (heightmap-last-index heightmap)] - (when (and (<= 0 x last) - (<= 0 y last)) - (heightmap-get heightmap x y)))) - - (defn heightmap-set! [heightmap x y val] - (.set heightmap x y val)) - - (defn heightmap-set-if-unset! [heightmap x y val] - (when (== 0 (heightmap-get heightmap x y)) - (heightmap-set! heightmap x y val))) - -Normalization: - - :::clojure - (defn normalize [heightmap] - (let [max (- Infinity) - min Infinity] - (do-ndarray-el el heightmap - (when (< max el) (set! max el)) - (when (> min el) (set! min el))) - (let [span (- max min)] - (do-ndarray [x y] heightmap - (heightmap-set! heightmap x y - (/ (- (heightmap-get heightmap x y) min) - span)))))) - -Creation: - - :::clojure - (defn make-heightmap [exponent] - (let [resolution (+ (Math.pow 2 exponent) 1)] - (let [heightmap (ndarray (new Float64Array (* resolution resolution)) - [resolution resolution])] - (set! heightmap.exponent exponent) - (set! heightmap.resolution resolution) - (set! heightmap.last (dec resolution)) - heightmap))) - -I'm not going to go through all the code here line-by-line because it's mostly -just a simple update of the last post. - -## Slicing Heightmaps - -Part of the Midpoint Displacement is "repeat the process on the four corner -squares of this one", and with ndarray we can make getting those corners much -simpler: - - :::clojure - (defn top-left-corner [heightmap] - (let [center (heightmap-center-index heightmap)] - (-> heightmap - (.lo 0 0) - (.hi (inc center) (inc center))))) - - (defn top-right-corner [heightmap] - (let [center (heightmap-center-index heightmap)] - (-> heightmap - (.lo center 0) - (.hi (inc center) (inc center))))) - - (defn bottom-left-corner [heightmap] - (let [center (heightmap-center-index heightmap)] - (-> heightmap - (.lo 0 center) - (.hi (inc center) (inc center))))) - - (defn bottom-right-corner [heightmap] - (let [center (heightmap-center-index heightmap)] - (-> heightmap - (.lo center center) - (.hi (inc center) (inc center))))) - -Each of these will return a "slice" of the underlying ndarray that looks and -acts like a fresh array (e.g. its indices start at `0`, `0`), but that uses the -appropriate part of the original array as the data storage. - -## Updating the Algorithm - -Now we can turn the algorithm into a recursive version. With the slicing -functions it's pretty simple. Initializing the corners is still trivial: - - :::clojure - (defn mpd-init-corners [heightmap] - (let [last (heightmap-last-index heightmap)] - (heightmap-set! heightmap 0 0 (rand)) - (heightmap-set! heightmap 0 last (rand)) - (heightmap-set! heightmap last 0 (rand)) - (heightmap-set! heightmap last last (rand)))) - -The meat of the algorithm looks long, but is mostly just calculating all the -appropriate numbers with readable names: - - :::clojure - (defn mpd-displace [heightmap spread spread-reduction] - (let [last (heightmap-last-index heightmap) - c (midpoint 0 last) - - ; Get the values of the corners - bottom-left (heightmap-get heightmap 0 0) - bottom-right (heightmap-get heightmap last 0) - top-left (heightmap-get heightmap 0 last) - top-right (heightmap-get heightmap last last) - - ; Calculate the averages for the points we're going to fill - top (average2 top-left top-right) - left (average2 bottom-left top-left) - bottom (average2 bottom-left bottom-right) - right (average2 bottom-right top-right) - center (average4 top left bottom right) - - next-spread (* spread spread-reduction)] - ; Set the four edge midpoint values - (heightmap-set-if-unset! heightmap c 0 (jitter bottom spread)) - (heightmap-set-if-unset! heightmap c last (jitter top spread)) - (heightmap-set-if-unset! heightmap 0 c (jitter left spread)) - (heightmap-set-if-unset! heightmap last c (jitter right spread)) - - ; Set the center value - (heightmap-set-if-unset! heightmap c c (jitter center spread)) - - ; Recurse on the four corners if necessary (3x3 is the base case) - (when-not (== 3 (heightmap-resolution heightmap)) - (mpd-displace (top-left-corner heightmap) next-spread spread-reduction) - (mpd-displace (top-right-corner heightmap) next-spread spread-reduction) - (mpd-displace (bottom-left-corner heightmap) next-spread spread-reduction) - (mpd-displace (bottom-right-corner heightmap) next-spread spread-reduction)))) - -The main wrapper function is simple: - - :::clojure - (defn midpoint-displacement [heightmap] - (let [initial-spread 0.3 - spread-reduction 0.55] - (mpd-init-corners heightmap) - (mpd-displace heightmap initial-spread spread-reduction) - (normalize heightmap))) - -## Result - -The result looks the same as before, but will generate the heightmaps a lot -faster because it's operating on a `Float64Array` instead of a vanilla JS array. - -
- -The code for these blog posts is a bit of a mess because I've been copy/pasting -to show the partially-completed demos. To fix that I've created a little -[single-page demo][ymir] with completed versions of the various algorithms you -can play with. [The code for that][ymir-code] should be a lot more readable -than the hacky code for these posts. - -[ymir]: http://ymir.stevelosh.com/ -[ymir-code]: http://bitbucket.org/sjl/ymir/ - - {% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2016/03/recursive-midpoint-displacement.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2016/03/recursive-midpoint-displacement.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,278 @@ ++++ +title = "Recursive Midpoint Displacement" +snip = "A cleaner version." +date = 2016-03-07T13:45:00Z +draft = false + ++++ + + + + + + +In the [last post][mpd] we looked at implementing the Midpoint Displacement +algorithm. I ended up doing the last step iteratively, which works, but isn't +the cleanest way to do it. Before moving on to other algorithms I wanted to +clean things up by using a handy library. + +The full series of posts so far: + +* [Midpoint Displacement](/blog/2016/02/midpoint-displacement/) +* [Recursive Midpoint Displacement](/blog/2016/03/recursive-midpoint-displacement/) +* [Diamond Square](/blog/2016/06/diamond-square/) + +[mpd]: /blog/2016/02/midpoint-displacement/ + +{{% toc %}} + +## Multi-Dimensional Arrays + +Last week when looking at something unrelated I came across the [ndarray][] +library ("nd" stands for "n-dimensional"). This is a little wrapper around +standard Javascript arrays to add easy multi-dimensional indexing. We're going +to to use `Float64Array` objects as the underlying storage because they're much +more efficient than the vanilla Javascript arrays and they're fairly well +supported. + +It also adds [slicing][], which is a lot like Common Lisp's [displaced +arrays][disp-arr]. It lets you create a new "array" object with a different +"shape" that doesn't have any actual storage of its own, but instead refers back +to the original array's data. This is perfect for implementing Midpoint +Displacement with recursion. + +[ndarray]: https://github.com/scijs/ndarray +[slicing]: https://github.com/scijs/ndarray#slicing +[disp-arr]: http://clhs.lisp.se/Body/26_glo_d.htm#displaced_array + +## Iteration + +We're still going to need to iterate over ndarrays at certain points (e.g. when +normalizing them) so let's make some helpful macros to do the annoying busywork +for us: + +```clojure +(defmacro do-ndarray [vars array-form & body] + (let [array-var (gensym "array") + build + (fn build [vars n] + (if (empty? vars) + `(do ~@body) + `(do-times ~(first vars) (aget (.-shape ~array-var) ~n) + ~(build (rest vars) (inc n)))))] + `(let [~array-var ~array-form] + ~(build vars 0)))) + +(defmacro do-ndarray-el [element array-form & body] + (let [index (gensym "index") + array (gensym "array")] + `(let [~array ~array-form] + (do-times ~index (.-length (.-data ~array)) + (let [~element (aget (.-data ~array) ~index)] + ~@body))))) +``` + +Now we can easily iterate over the indices: + +```clojure +(do-ndarray [x y] my-ndarray + (console.log "Array[" x "][" y "] is: " + (.get my-ndarray x y))) +``` + +Or just over the items if we don't need their indices: + +```clojure +(do-ndarray-el item my-ndarray + (console.log item)) +``` + +These macros should work for ndarrays of any number of dimensions, and will +compile into ugly but fast Javascript `for` loops. + +## Updating the Heightmaps + +To start we'll need to update the heightmap functions to work with ndarrays +instead of the normal JS arrays. + +Start with a few functions to calculate sizes/incides: + +```clojure +(defn heightmap-resolution [heightmap] + (aget heightmap.shape 0)) + +(defn heightmap-last-index [heightmap] + (dec (heightmap-resolution heightmap))) + +(defn heightmap-center-index [heightmap] + (midpoint 0 (heightmap-last-index heightmap))) +``` + +Support for reading/writing: + +```clojure +(defn heightmap-get [heightmap x y] + (.get heightmap x y)) + +(defn heightmap-get-safe [heightmap x y] + (let [last (heightmap-last-index heightmap)] + (when (and (<= 0 x last) + (<= 0 y last)) + (heightmap-get heightmap x y)))) + +(defn heightmap-set! [heightmap x y val] + (.set heightmap x y val)) + +(defn heightmap-set-if-unset! [heightmap x y val] + (when (== 0 (heightmap-get heightmap x y)) + (heightmap-set! heightmap x y val))) +``` + +Normalization: + +```clojure +(defn normalize [heightmap] + (let [max (- Infinity) + min Infinity] + (do-ndarray-el el heightmap + (when (< max el) (set! max el)) + (when (> min el) (set! min el))) + (let [span (- max min)] + (do-ndarray [x y] heightmap + (heightmap-set! heightmap x y + (/ (- (heightmap-get heightmap x y) min) + span)))))) +``` + +Creation: + +```clojure +(defn make-heightmap [exponent] + (let [resolution (+ (Math.pow 2 exponent) 1)] + (let [heightmap (ndarray (new Float64Array (* resolution resolution)) + [resolution resolution])] + (set! heightmap.exponent exponent) + (set! heightmap.resolution resolution) + (set! heightmap.last (dec resolution)) + heightmap))) +``` + +I'm not going to go through all the code here line-by-line because it's mostly +just a simple update of the last post. + +## Slicing Heightmaps + +Part of the Midpoint Displacement is "repeat the process on the four corner +squares of this one", and with ndarray we can make getting those corners much +simpler: + +```clojure +(defn top-left-corner [heightmap] + (let [center (heightmap-center-index heightmap)] + (-> heightmap + (.lo 0 0) + (.hi (inc center) (inc center))))) + +(defn top-right-corner [heightmap] + (let [center (heightmap-center-index heightmap)] + (-> heightmap + (.lo center 0) + (.hi (inc center) (inc center))))) + +(defn bottom-left-corner [heightmap] + (let [center (heightmap-center-index heightmap)] + (-> heightmap + (.lo 0 center) + (.hi (inc center) (inc center))))) + +(defn bottom-right-corner [heightmap] + (let [center (heightmap-center-index heightmap)] + (-> heightmap + (.lo center center) + (.hi (inc center) (inc center))))) +``` + +Each of these will return a "slice" of the underlying ndarray that looks and +acts like a fresh array (e.g. its indices start at `0`, `0`), but that uses the +appropriate part of the original array as the data storage. + +## Updating the Algorithm + +Now we can turn the algorithm into a recursive version. With the slicing +functions it's pretty simple. Initializing the corners is still trivial: + +```clojure +(defn mpd-init-corners [heightmap] + (let [last (heightmap-last-index heightmap)] + (heightmap-set! heightmap 0 0 (rand)) + (heightmap-set! heightmap 0 last (rand)) + (heightmap-set! heightmap last 0 (rand)) + (heightmap-set! heightmap last last (rand)))) +``` + +The meat of the algorithm looks long, but is mostly just calculating all the +appropriate numbers with readable names: + +```clojure +(defn mpd-displace [heightmap spread spread-reduction] + (let [last (heightmap-last-index heightmap) + c (midpoint 0 last) + + ; Get the values of the corners + bottom-left (heightmap-get heightmap 0 0) + bottom-right (heightmap-get heightmap last 0) + top-left (heightmap-get heightmap 0 last) + top-right (heightmap-get heightmap last last) + + ; Calculate the averages for the points we're going to fill + top (average2 top-left top-right) + left (average2 bottom-left top-left) + bottom (average2 bottom-left bottom-right) + right (average2 bottom-right top-right) + center (average4 top left bottom right) + + next-spread (* spread spread-reduction)] + ; Set the four edge midpoint values + (heightmap-set-if-unset! heightmap c 0 (jitter bottom spread)) + (heightmap-set-if-unset! heightmap c last (jitter top spread)) + (heightmap-set-if-unset! heightmap 0 c (jitter left spread)) + (heightmap-set-if-unset! heightmap last c (jitter right spread)) + + ; Set the center value + (heightmap-set-if-unset! heightmap c c (jitter center spread)) + + ; Recurse on the four corners if necessary (3x3 is the base case) + (when-not (== 3 (heightmap-resolution heightmap)) + (mpd-displace (top-left-corner heightmap) next-spread spread-reduction) + (mpd-displace (top-right-corner heightmap) next-spread spread-reduction) + (mpd-displace (bottom-left-corner heightmap) next-spread spread-reduction) + (mpd-displace (bottom-right-corner heightmap) next-spread spread-reduction)))) +``` + +The main wrapper function is simple: + +```clojure +(defn midpoint-displacement [heightmap] + (let [initial-spread 0.3 + spread-reduction 0.55] + (mpd-init-corners heightmap) + (mpd-displace heightmap initial-spread spread-reduction) + (normalize heightmap))) +``` + +## Result + +The result looks the same as before, but will generate the heightmaps a lot +faster because it's operating on a `Float64Array` instead of a vanilla JS array. + +
+ +The code for these blog posts is a bit of a mess because I've been copy/pasting +to show the partially-completed demos. To fix that I've created a little +[single-page demo][ymir] with completed versions of the various algorithms you +can play with. [The code for that][ymir-code] should be a lot more readable +than the hacky code for these posts. + +[ymir]: http://ymir.stevelosh.com/ +[ymir-code]: http://bitbucket.org/sjl/ymir/ + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2016/06/diamond-square.html --- a/content/blog/2016/06/diamond-square.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,476 +0,0 @@ - {% extends "_post.html" %} - - {% load mathjax %} - - {% hyde - title: "Terrain Generation with Diamond Square" - snip: "Improving on Midpoint Displacement." - created: 2016-06-27 13:35:00 - %} - - {% block extra_js %} - - - - {% endblock extra_js %} - - {% block article %} - -In the last two posts we looked at implementing the Midpoint Displacement -algorithm for procedurally generating terrain. Today we're going to look at -a similar algorithm called Diamond Square that fixes some problems with Midpoint -Displacement. - -The full series of posts so far: - -* [Midpoint Displacement](/blog/2016/02/midpoint-displacement/) -* [Recursive Midpoint Displacement](/blog/2016/03/recursive-midpoint-displacement/) -* [Diamond Square](/blog/2016/06/diamond-square/) - -Midpoint Displacement is simple and fast, but it has some flaws. If you look at -the end result you'll probably start to notice "seams" in the terrain that -appear on perfectly square chunks. This happens because when you're calculating -the midpoints of the edges of each square you're only averaging two sources of -information. - -[Diamond Square][] is an algorithm that works a lot like Midpoint Displacement, -but it adds an extra step to ensure that (almost) every point uses *four* -sources of data. This reduces the visual artifacts a lot without much extra -effort. - -Let's [draw the owl][owl]. - -[Diamond Square]: https://en.wikipedia.org/wiki/Diamond-square_algorithm -[owl]: https://i.imgur.com/RadSf.jpg - -[TOC] - -## Overview - -The high-level description of Diamond Square goes something like this: - -1. Initialize the corners to random values. -2. Set the center of the heightmap to the average of the corners (plus jitter). -3. Set the midpoints of the edges of the heightmap to the average of the four - points on the "diamond" around them (plus jitter). -4. Repeat steps 2-4 on successively smaller chunks of the heightmap until you - bottom out at 3x3 chunks. - -## Initialization - -Initialization is pretty simple: - - :::clojure - (defn ds-init-corners [heightmap] - (let [last (heightmap-last-index heightmap)] - (heightmap-set! heightmap 0 0 (rand)) - (heightmap-set! heightmap 0 last (rand)) - (heightmap-set! heightmap last 0 (rand)) - (heightmap-set! heightmap last last (rand)))) - -Just set the corners to random values, exactly like we did in Midpoint -Displacement. - -
- -## Square - -In the "square" step of Diamond Square, we take a square region of the heightmap -and set the center of it to the average of the four corners (plus jitter). - -
-           Column
-         0 1 2 3 4         0 1 2 3 4         0 1 2 3 4
-        ┌─┬─┬─┬─┬─┐       ┌─┬─┬─┬─┬─┐       ┌─┬─┬─┬─┬─┐
-      0 │1│ │ │ │8│     0 │1│ │ │ │8│     0 │1│ │ │ │8│
-        ├─┼─┼─┼─┼─┤       ├─╲─┼─┼─╱─┤       ├─╲─┼─┼─╱─┤
-      1 │ │ │ │ │ │     1 │ │╲│ │╱│ │     1 │ │╲│ │╱│ │
-    R   ├─┼─┼─┼─┼─┤       ├─┼─◢─◣─┼─┤       ├─┼─◢─◣─┼─┤
-    o 2 │ │ │◉│ │ │     2 │ │ │◉│ │ │     2 │ │ │3│ │ │
-    w   ├─┼─┼─┼─┼─┤       ├─┼─◥─◤─┼─┤       ├─┼─◥─◤─┼─┤
-      3 │ │ │ │ │ │     3 │ │╱│ │╲│ │     3 │ │╱│ │╲│ │
-        ├─┼─┼─┼─┼─┤       ├─╱─┼─┼─╲─┤       ├─╱─┼─┼─╲─┤
-      4 │0│ │ │ │3│     4 │0│ │ │ │3│     4 │0│ │ │ │3│
-        └─┴─┴─┴─┴─┘       └─┴─┴─┴─┴─┘       └─┴─┴─┴─┴─┘
-
- -The Wikipedia article calls this the "diamond" step, but that seems backwards to -me. We're operating on a *square* region of the heightmap, so this should be -the *square* step. - -Since we're no longer working with heightmap "slices" like we were in the -previous post, we'll need a way to specify the square chunk of the heightmap -we're working on. I chose to use a center point (`x` and `y` coordinates) and -a radius: - - :::clojure - (defn ds-square [heightmap x y radius spread] - (let [new-height - (jitter - (average4 - (heightmap-get heightmap (- x radius) (- y radius)) - (heightmap-get heightmap (- x radius) (+ y radius)) - (heightmap-get heightmap (+ x radius) (- y radius)) - (heightmap-get heightmap (+ x radius) (+ y radius))) - spread)] - (heightmap-set! heightmap x y new-height))) - -
- -## Diamond - -In the "diamond" step of Diamond Square, we take a diamond-shaped region and set -the center of it to the average of the four corners: - -
-               Column
-         0 1 2 3 4 5 6 7 8        0 1 2 3 4 5 6 7 8
-        ┌─┬─┬─┬─┬─┬─┬─┬─┬─┐      ┌─┬─┬─┬─┬─┬─┬─┬─┬─┐
-      0 │1│ │ │ │3│ │ │ │3│    0 │1│ │ │ │3│ │ │ │3│
-        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─┼─╱─╲─┼─┼─┼─┤
-      1 │ │ │ │ │ │ │ │ │ │    1 │ │ │ │╱│ │╲│ │ │ │
-    R   ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─╱─┼─┼─╲─┼─┼─┤
-    o 2 │ │ │3│ │ │ │4│ │ │    2 │ │ │3│ │◉│ │4│ │ │
-    w   ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─╲─┼─┼─╱─┼─┼─┤
-      3 │ │ │ │ │ │ │ │ │ │    3 │ │ │ │╲│ │╱│ │ │ │
-        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─┼─╲─╱─┼─┼─┼─┤
-      4 │5│ │ │ │5│ │ │ │5│    4 │5│ │ │ │5│ │ │ │5│
-        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
-      5 │ │ │ │ │ │ │ │ │ │    5 │ │ │ │ │ │ │ │ │ │
-        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
-      6 │ │ │7│ │ │ │6│ │ │    6 │ │ │7│ │ │ │6│ │ │
-        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
-      7 │ │ │ │ │ │ │ │ │ │    7 │ │ │ │ │ │ │ │ │ │
-        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
-      8 │9│ │ │ │7│ │ │ │7│    8 │9│ │ │ │7│ │ │ │7│
-        └─┴─┴─┴─┴─┴─┴─┴─┴─┘      └─┴─┴─┴─┴─┴─┴─┴─┴─┘
-
-               Column
-         0 1 2 3 4 5 6 7 8        0 1 2 3 4 5 6 7 8
-        ┌─┬─┬─┬─┬─┬─┬─┬─┬─┐      ┌─┬─┬─┬─┬─┬─┬─┬─┬─┐
-      0 │1│ │ │ │3│ │ │ │3│    0 │1│ │ │ │3│ │ │ │3│
-        ├─┼─┼─┼─╱┬╲─┼─┼─┼─┤      ├─┼─┼─┼─╱┬╲─┼─┼─┼─┤
-      1 │ │ │ │╱│││╲│ │ │ │    1 │ │ │ │╱│││╲│ │ │ │
-    R   ├─┼─┼─╱─┼▼┼─╲─┼─┼─┤      ├─┼─┼─╱─┼▼┼─╲─┼─┼─┤
-    o 2 │ │ │3├─▶◉◀─┤4│ │ │    2 │ │ │3├─▶4◀─┤4│ │ │
-    w   ├─┼─┼─╲─┼▲┼─╱─┼─┼─┤      ├─┼─┼─╲─┼▲┼─╱─┼─┼─┤
-      3 │ │ │ │╲│││╱│ │ │ │    3 │ │ │ │╲│││╱│ │ │ │
-        ├─┼─┼─┼─╲┴╱─┼─┼─┼─┤      ├─┼─┼─┼─╲┴╱─┼─┼─┼─┤
-      4 │5│ │ │ │5│ │ │ │5│    4 │5│ │ │ │5│ │ │ │5│
-        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
-      5 │ │ │ │ │ │ │ │ │ │    5 │ │ │ │ │ │ │ │ │ │
-        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
-      6 │ │ │7│ │ │ │6│ │ │    6 │ │ │7│ │ │ │6│ │ │
-        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
-      7 │ │ │ │ │ │ │ │ │ │    7 │ │ │ │ │ │ │ │ │ │
-        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
-      8 │9│ │ │ │7│ │ │ │7│    8 │9│ │ │ │7│ │ │ │7│
-        └─┴─┴─┴─┴─┴─┴─┴─┴─┘      └─┴─┴─┴─┴─┴─┴─┴─┴─┘
-
- -Once again we'll use a center point and radius to specify the region inside the -heightmap: - - :::clojure - (defn ds-diamond [heightmap x y radius spread] - (let [new-height - (jitter - (safe-average - (heightmap-get-safe heightmap (- x radius) y) - (heightmap-get-safe heightmap (+ x radius) y) - (heightmap-get-safe heightmap x (- y radius)) - (heightmap-get-safe heightmap x (+ y radius))) - spread)] - (heightmap-set! heightmap x y new-height))) - -
- -## Edge Cases - -You might have noticed that we used `heightmap-get-safe` and `safe-average` in -the `ds-diamond` code. This is to handle the diamonds on the edge of the -heightmap. Those diamonds only have three points to average: - -
-           Column
-         0 1 2 3 4        0 1 2 3 4
-        ┌─┬─┬─┬─┬─┐      ┌─┬─┬─┬─┬─┐
-      0 │1│ │ │ │8│    0 │1│ │ │ │8│
-        ├─┼─┼─┼─┼─┤      ├─┼─┼─┼─╱┬╲
-      1 │ │ │ │ │ │    1 │ │ │ │╱│││╲
-    R   ├─┼─┼─┼─┼─┤      ├─┼─┼─╱─┼▼┤ ╲
-    o 2 │ │ │3│ │ │    2 │ │ │3├─▶◉◀──?
-    w   ├─┼─┼─┼─┼─┤      ├─┼─┼─╲─┼▲┤ ╱
-      3 │ │ │ │ │ │    3 │ │ │ │╲│││╱
-        ├─┼─┼─┼─┼─┤      ├─┼─┼─┼─╲┴╱
-      4 │0│ │ │ │3│    4 │0│ │ │ │3│
-        └─┴─┴─┴─┴─┘      └─┴─┴─┴─┴─┘
-
- -So we just ignore the missing point: - - :::clojure - (defn safe-average [a b c d] - (let [total 0 count 0] - (when a (add! total a) (inc! count)) - (when b (add! total b) (inc! count)) - (when c (add! total c) (inc! count)) - (when d (add! total d) (inc! count)) - (/ total count))) - - (defn heightmap-get-safe [heightmap x y] - (let [last (heightmap-last-index heightmap)] - (when (and (<= 0 x last) - (<= 0 y last)) - (heightmap-get heightmap x y)))) - -Technically this means they have less information to work with than the rest of -the points, and you might see some slight artifacts. But in practice it's not -too bad, and it's a lot nicer than midpoint displacement where *every* line step -uses only *two* points. - -Another way to handle this is to also wrap those edge coordinates around and -pull the point from the other side of the heightmap. This is a bit more work -but gives you an extra benefit: the heightmap will be tileable (**update**: like -everything where someone doesn't show you running code, it's [not actually that -simple](/blog/2016/08/lisp-jam-postmortem/#tiling-diamond-square)). - -## Iteration - -Unfortunately we can't use recursion to iterate for Diamond Square like we did -for Midpoint Displacement. We have to interleave the diamond and square steps, -performing each round of squares on the *entire* map before moving on to a round -of diamonds. If we don't, bad things will happen. - -Let's look at an example to see why. First we perform the initial square step: - -
-           Column
-         0 1 2 3 4        0 1 2 3 4
-        ┌─┬─┬─┬─┬─┐      ┌─┬─┬─┬─┬─┐
-      0 │1│ │ │ │8│    0 │1│ │ │ │8│
-        ├─┼─┼─┼─┼─┤      ├─╲─┼─┼─╱─┤
-      1 │ │ │ │ │ │    1 │ │╲│ │╱│ │
-    R   ├─┼─┼─┼─┼─┤      ├─┼─◢─◣─┼─┤
-    o 2 │ │ │ │ │ │    2 │ │ │3│ │ │
-    w   ├─┼─┼─┼─┼─┤      ├─┼─◥─◤─┼─┤
-      3 │ │ │ │ │ │    3 │ │╱│ │╲│ │
-        ├─┼─┼─┼─┼─┤      ├─╱─┼─┼─╲─┤
-      4 │0│ │ │ │3│    4 │0│ │ │ │3│
-        └─┴─┴─┴─┴─┘      └─┴─┴─┴─┴─┘
-
- -That's fine. Then we recurse into the first of the four diamonds: - -
-           Column
-         0 1 2 3 4        0 1 2 3 4
-        ┌─┬─┬─┬─┬─┐      ┌─┬─┬─┬─┬─┐
-      0 │1│ │ │ │8│    0 │1├─▶4◀─┤8│
-        ├─┼─┼─┼─┼─┤      ├─╲─┼▲┼─╱─┤
-      1 │ │ │ │ │ │    1 │ │╲│││╱│ │
-    R   ├─┼─┼─┼─┼─┤      ├─┼─╲┴╱─┼─┤
-    o 2 │ │ │3│ │ │    2 │ │ │3│ │ │
-    w   ├─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┤
-      3 │ │ │ │ │ │    3 │ │ │ │ │ │
-        ├─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┤
-      4 │0│ │ │ │3│    4 │0│ │ │ │3│
-        └─┴─┴─┴─┴─┘      └─┴─┴─┴─┴─┘
-
- -Everything's still good. Then we recurse into the square: - -
-            Column
-         0 1 2 3 4        0 1 2 3 4
-        ┌─┬─┬─┬─┬─┐      ╔═════╗─┬─┐
-      0 │1│ │4│ │8│    0 ║1│ │4║ │8│
-        ├─┼─┼─┼─┼─┤      ║─◢─◣─║─┼─┤
-      1 │ │ │ │ │ │    1 ║ │ │ ║ │ │
-    R   ├─┼─┼─┼─┼─┤      ║─◥─◤─║─┼─┤
-    o 2 │ │ │3│ │ │    2 ║?│ │3║ │ │
-    w   ├─┼─┼─┼─┼─┤      ╚═════╝─┼─┤
-      3 │ │ │ │ │ │    3 │ │ │ │ │ │
-        ├─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┤
-      4 │0│ │ │ │3│    4 │0│ │ │ │3│
-        └─┴─┴─┴─┴─┘      └─┴─┴─┴─┴─┘
-
- -But wait, there's a corner missing! We needed to complete that left-hand -diamond step to get it, but that's still waiting on the stack for this line of -recursion to bottom out. - -### A "Strided" Iteration Construct - -So we need to use normal iteration. We can start by writing the code to do -a single round of diamonds or squares on the heightmap. First we'll make a nice -looping construct called `do-stride`: - - :::clojure - (defmacro do-stride - [varnames start-form end-form stride-form & body] - (let [stride (gensym "stride") - start (gensym "start") - end (gensym "end") - build (fn build [vars] - (if (empty? vars) - `(do ~@body) - (let [varname (first vars)] - `(loop [~varname ~start] - (when (< ~varname ~end) - ~(build (rest vars)) - (recur (+ ~varname ~stride)))))))] - ; Fix the numbers once outside the nested loops, - ; and then build the guts. - `(let [~start ~start-form - ~end ~end-form - ~stride ~stride-form] - ~(build varnames)))) - -This scary-looking macro builds the nested looping structure we'll need to -perform the iteration for our diamonds and squares. It abstracts away the -tedium of writing out nested `for` loops so we can focus on the rest of the -algorithm. - -`do-stride` takes a list of variables to bind, a number to start at, a number to -end before, and a body. For each variable it starts at `start`, runs `body`, -increments by `stride`, and loops until the value is at or above `end`. For -example: - -* `(do-stride [x] 0 5 1 (console.log x))` will log `0 1 2 3 4`. -* `(do-stride [x] 0 5 2 (console.log x))` will log `0 2 4`. -* `(do-stride [x y] 0 3 2 (console.log [x y]))` will log `[0, 0] [0, 2] [2, 0] - [2, 2]`. - -### Square Iteration - -Now that we've got this, we can iterate our square step over the heightmap for -a specific radius: - - :::clojure - (defn ds-squares [heightmap radius spread] - (do-stride [x y] radius (heightmap-resolution heightmap) (* 2 radius) - (ds-square heightmap x y radius spread))) - -We use `do-stride` to start at the radius and loop until we fall off the -heightmap, stepping by *twice* the radius each time. - -Because our heightmaps are always square we can use the same values for both -dimensions. `do-stride` can take zero or more variables and will just do the -right thing, so we don't need to worry about it. - -The stride is double the radius because as we step between squares we move over -the right half of the first *and* the left half of the next: - -
-               Column
-          0 1 2 3 4 5 6 7 8        0 1 2 3 4 5 6 7 8
-         ╔═════════╗─┬─┬─┬─┐      ╔═════════╗─┬─┬─┬─┐
-       0 ║1│ │ │ │3║ │ │ │3│    0 ║1│ │ │ │3║ │ │ │3│
-         ║─┼─┼─┼─┼─║─┼─┼─┼─┤      ║─┼─┼─┼─┼─║─┼─┼─┼─┤
-       1 ║ │ │ │ │ ║ │ │ │ │    1 ║ │ │ │ │ ║ │ │ │ │
-    R    ║─┼─┼─┼─┼─║─┼─┼─┼─┤     radius ┼─┼─║─┼─┼─┼─┤
-    o  2 ║ │ │◉│ │ ║ │◉│ │ │    2 ║◀━━▶◉◀━━━━━▶◉│ │ │
-    w    ║─┼─┼─┼─┼─║─┼─┼─┼─┤      ║─┼─┼─ stride ┼─┼─┤
-       3 ║ │ │ │ │ ║ │ │ │ │    3 ║ │ │ │ │ ║ │ │ │ │
-         ║─┼─┼─┼─┼─║─┼─┼─┼─┤      ║─┼─┼─┼─┼─║─┼─┼─┼─┤
-       4 ║5│ │ │ │5║ │ │ │5│    4 ║5│ │ │ │5║ │ │ │5│
-         ╚═════════╝─┼─┼─┼─┤      ╚═════════╝─┼─┼─┼─┤
-       5 │ │ │ │ │ │ │ │ │ │    5 │ │ │ │ │ │ │ │ │ │
-         ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
-       6 │ │ │◉│ │ │ │◉│ │ │    6 │ │ │◉│ │ │ │◉│ │ │
-         ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
-       7 │ │ │ │ │ │ │ │ │ │    7 │ │ │ │ │ │ │ │ │ │
-         ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
-       8 │9│ │ │ │7│ │ │ │7│    8 │9│ │ │ │7│ │ │ │7│
-         └─┴─┴─┴─┴─┴─┴─┴─┴─┘      └─┴─┴─┴─┴─┴─┴─┴─┴─┘
-
- -### Diamond Iteration - -The diamonds are a little more complicated. - -First of all, in the `y` dimension we need to start at `0` instead of `radius` -(these will be the top and bottom edge cases we looked at). Also, in the `x` -direction all the even iterations need to start at `radius`, while the odd -iterations should start at `0`. - -Hopefully a picture will make it a bit more clear: - -
-               Column
-         0 1 2 3 4 5 6 7 8
-        ┌─┬─┬─┬─┬─┬─┬─┬─┬─┐
-      0 │1│ │◉│ │3│ │◉│ │3│ y iteration: 0, start at x = radius
-        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
-      1 │ │ │ │ │ │ │ │ │ │
-    R   ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
-    o 2 │◉│ │3│ │◉│ │4│ │◉│ y iteration: 1, start at x = 0
-    w   ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
-      3 │ │ │ │ │ │ │ │ │ │
-        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
-      4 │5│ │◉│ │5│ │◉│ │5│ y iteration: 2, start at x = radius
-        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
-      5 │ │ │ │ │ │ │ │ │ │
-        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
-      6 │◉│ │7│ │◉│ │6│ │◉│ y iteration: 3, start at x = 0
-        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
-      7 │ │ │ │ │ │ │ │ │ │
-        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
-      8 │9│ │◉│ │7│ │◉│ │7│ y iteration: 4, start at x = radius
-        └─┴─┴─┴─┴─┴─┴─┴─┴─┘
-
- -The code for the diamonds is the hairiest bit of the algorithm. Make sure you -understand it before moving on. `shift` will be `radius` for even iterations -and `0` for odd ones: - - :::clojure - (defn ds-diamonds [heightmap radius spread] - (let [size (heightmap-resolution heightmap)] - (do-stride [y] 0 size radius - (let [shift (if (even? (/ y radius)) radius 0)] - (do-stride [x] shift size (* 2 radius) - (ds-diamond heightmap x y radius spread)))))) - -### Top-Level Iteration - -Finally we can put it all together. We initialize the corners and starting -values, then loop over smaller and smaller radii and just call `ds-squares` and -`ds-diamonds` to do the heavy lifting: - - :::clojure - (defn diamond-square [heightmap] - (let [initial-spread 0.3 - spread-reduction 0.5 - center (heightmap-center-index heightmap) - size (aget heightmap.shape 0)] - (ds-init-corners heightmap) - (loop [radius center - spread initial-spread] - (when (>= radius 1) - (ds-squares heightmap radius spread) - (ds-diamonds heightmap radius spread) - (recur (/ radius 2) - (* spread spread-reduction)))) - (sanitize heightmap))) - -## Result - -The end result looks like this: - -
- -It looks a lot like the result from Midpoint Displacement, but without the ugly -seams around the square chunks. - -The code for these blog posts is a bit of a mess because I've been copy/pasting -to show the partially-completed demos after each step. I've created a little -[single-page demo][ymir] with completed versions of the various algorithms you -can play with, and [that code][ymir-code] should be a lot more readable than the -hacky code for these posts. - -[ymir]: http://ymir.stevelosh.com/ -[ymir-code]: http://bitbucket.org/sjl/ymir/ - - {% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2016/06/diamond-square.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2016/06/diamond-square.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,477 @@ ++++ +title = "Terrain Generation with Diamond Square" +snip = "Improving on Midpoint Displacement." +date = 2016-06-27T13:35:00Z +draft = false + ++++ + + + + + +In the last two posts we looked at implementing the Midpoint Displacement +algorithm for procedurally generating terrain. Today we're going to look at +a similar algorithm called Diamond Square that fixes some problems with Midpoint +Displacement. + +The full series of posts so far: + +* [Midpoint Displacement](/blog/2016/02/midpoint-displacement/) +* [Recursive Midpoint Displacement](/blog/2016/03/recursive-midpoint-displacement/) +* [Diamond Square](/blog/2016/06/diamond-square/) + +Midpoint Displacement is simple and fast, but it has some flaws. If you look at +the end result you'll probably start to notice "seams" in the terrain that +appear on perfectly square chunks. This happens because when you're calculating +the midpoints of the edges of each square you're only averaging two sources of +information. + +[Diamond Square][] is an algorithm that works a lot like Midpoint Displacement, +but it adds an extra step to ensure that (almost) every point uses *four* +sources of data. This reduces the visual artifacts a lot without much extra +effort. + +Let's [draw the owl][owl]. + +[Diamond Square]: https://en.wikipedia.org/wiki/Diamond-square_algorithm +[owl]: https://i.imgur.com/RadSf.jpg + +{{% toc %}} + +## Overview + +The high-level description of Diamond Square goes something like this: + +1. Initialize the corners to random values. +2. Set the center of the heightmap to the average of the corners (plus jitter). +3. Set the midpoints of the edges of the heightmap to the average of the four + points on the "diamond" around them (plus jitter). +4. Repeat steps 2-4 on successively smaller chunks of the heightmap until you + bottom out at 3x3 chunks. + +## Initialization + +Initialization is pretty simple: + +```clojure +(defn ds-init-corners [heightmap] + (let [last (heightmap-last-index heightmap)] + (heightmap-set! heightmap 0 0 (rand)) + (heightmap-set! heightmap 0 last (rand)) + (heightmap-set! heightmap last 0 (rand)) + (heightmap-set! heightmap last last (rand)))) +``` + +Just set the corners to random values, exactly like we did in Midpoint +Displacement. + +
+ +## Square + +In the "square" step of Diamond Square, we take a square region of the heightmap +and set the center of it to the average of the four corners (plus jitter). + +
+           Column
+         0 1 2 3 4         0 1 2 3 4         0 1 2 3 4
+        ┌─┬─┬─┬─┬─┐       ┌─┬─┬─┬─┬─┐       ┌─┬─┬─┬─┬─┐
+      0 │1│ │ │ │8│     0 │1│ │ │ │8│     0 │1│ │ │ │8│
+        ├─┼─┼─┼─┼─┤       ├─╲─┼─┼─╱─┤       ├─╲─┼─┼─╱─┤
+      1 │ │ │ │ │ │     1 │ │╲│ │╱│ │     1 │ │╲│ │╱│ │
+    R   ├─┼─┼─┼─┼─┤       ├─┼─◢─◣─┼─┤       ├─┼─◢─◣─┼─┤
+    o 2 │ │ │◉│ │ │     2 │ │ │◉│ │ │     2 │ │ │3│ │ │
+    w   ├─┼─┼─┼─┼─┤       ├─┼─◥─◤─┼─┤       ├─┼─◥─◤─┼─┤
+      3 │ │ │ │ │ │     3 │ │╱│ │╲│ │     3 │ │╱│ │╲│ │
+        ├─┼─┼─┼─┼─┤       ├─╱─┼─┼─╲─┤       ├─╱─┼─┼─╲─┤
+      4 │0│ │ │ │3│     4 │0│ │ │ │3│     4 │0│ │ │ │3│
+        └─┴─┴─┴─┴─┘       └─┴─┴─┴─┴─┘       └─┴─┴─┴─┴─┘
+
+ +The Wikipedia article calls this the "diamond" step, but that seems backwards to +me. We're operating on a *square* region of the heightmap, so this should be +the *square* step. + +Since we're no longer working with heightmap "slices" like we were in the +previous post, we'll need a way to specify the square chunk of the heightmap +we're working on. I chose to use a center point (`x` and `y` coordinates) and +a radius: + +```clojure +(defn ds-square [heightmap x y radius spread] + (let [new-height + (jitter + (average4 + (heightmap-get heightmap (- x radius) (- y radius)) + (heightmap-get heightmap (- x radius) (+ y radius)) + (heightmap-get heightmap (+ x radius) (- y radius)) + (heightmap-get heightmap (+ x radius) (+ y radius))) + spread)] + (heightmap-set! heightmap x y new-height))) +``` + +
+ +## Diamond + +In the "diamond" step of Diamond Square, we take a diamond-shaped region and set +the center of it to the average of the four corners: + +
+               Column
+         0 1 2 3 4 5 6 7 8        0 1 2 3 4 5 6 7 8
+        ┌─┬─┬─┬─┬─┬─┬─┬─┬─┐      ┌─┬─┬─┬─┬─┬─┬─┬─┬─┐
+      0 │1│ │ │ │3│ │ │ │3│    0 │1│ │ │ │3│ │ │ │3│
+        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─┼─╱─╲─┼─┼─┼─┤
+      1 │ │ │ │ │ │ │ │ │ │    1 │ │ │ │╱│ │╲│ │ │ │
+    R   ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─╱─┼─┼─╲─┼─┼─┤
+    o 2 │ │ │3│ │ │ │4│ │ │    2 │ │ │3│ │◉│ │4│ │ │
+    w   ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─╲─┼─┼─╱─┼─┼─┤
+      3 │ │ │ │ │ │ │ │ │ │    3 │ │ │ │╲│ │╱│ │ │ │
+        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─┼─╲─╱─┼─┼─┼─┤
+      4 │5│ │ │ │5│ │ │ │5│    4 │5│ │ │ │5│ │ │ │5│
+        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
+      5 │ │ │ │ │ │ │ │ │ │    5 │ │ │ │ │ │ │ │ │ │
+        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
+      6 │ │ │7│ │ │ │6│ │ │    6 │ │ │7│ │ │ │6│ │ │
+        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
+      7 │ │ │ │ │ │ │ │ │ │    7 │ │ │ │ │ │ │ │ │ │
+        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
+      8 │9│ │ │ │7│ │ │ │7│    8 │9│ │ │ │7│ │ │ │7│
+        └─┴─┴─┴─┴─┴─┴─┴─┴─┘      └─┴─┴─┴─┴─┴─┴─┴─┴─┘
+
+               Column
+         0 1 2 3 4 5 6 7 8        0 1 2 3 4 5 6 7 8
+        ┌─┬─┬─┬─┬─┬─┬─┬─┬─┐      ┌─┬─┬─┬─┬─┬─┬─┬─┬─┐
+      0 │1│ │ │ │3│ │ │ │3│    0 │1│ │ │ │3│ │ │ │3│
+        ├─┼─┼─┼─╱┬╲─┼─┼─┼─┤      ├─┼─┼─┼─╱┬╲─┼─┼─┼─┤
+      1 │ │ │ │╱│││╲│ │ │ │    1 │ │ │ │╱│││╲│ │ │ │
+    R   ├─┼─┼─╱─┼▼┼─╲─┼─┼─┤      ├─┼─┼─╱─┼▼┼─╲─┼─┼─┤
+    o 2 │ │ │3├─▶◉◀─┤4│ │ │    2 │ │ │3├─▶4◀─┤4│ │ │
+    w   ├─┼─┼─╲─┼▲┼─╱─┼─┼─┤      ├─┼─┼─╲─┼▲┼─╱─┼─┼─┤
+      3 │ │ │ │╲│││╱│ │ │ │    3 │ │ │ │╲│││╱│ │ │ │
+        ├─┼─┼─┼─╲┴╱─┼─┼─┼─┤      ├─┼─┼─┼─╲┴╱─┼─┼─┼─┤
+      4 │5│ │ │ │5│ │ │ │5│    4 │5│ │ │ │5│ │ │ │5│
+        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
+      5 │ │ │ │ │ │ │ │ │ │    5 │ │ │ │ │ │ │ │ │ │
+        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
+      6 │ │ │7│ │ │ │6│ │ │    6 │ │ │7│ │ │ │6│ │ │
+        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
+      7 │ │ │ │ │ │ │ │ │ │    7 │ │ │ │ │ │ │ │ │ │
+        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
+      8 │9│ │ │ │7│ │ │ │7│    8 │9│ │ │ │7│ │ │ │7│
+        └─┴─┴─┴─┴─┴─┴─┴─┴─┘      └─┴─┴─┴─┴─┴─┴─┴─┴─┘
+
+ +Once again we'll use a center point and radius to specify the region inside the +heightmap: + +```clojure +(defn ds-diamond [heightmap x y radius spread] + (let [new-height + (jitter + (safe-average + (heightmap-get-safe heightmap (- x radius) y) + (heightmap-get-safe heightmap (+ x radius) y) + (heightmap-get-safe heightmap x (- y radius)) + (heightmap-get-safe heightmap x (+ y radius))) + spread)] + (heightmap-set! heightmap x y new-height))) +``` + +
+ +## Edge Cases + +You might have noticed that we used `heightmap-get-safe` and `safe-average` in +the `ds-diamond` code. This is to handle the diamonds on the edge of the +heightmap. Those diamonds only have three points to average: + +
+           Column
+         0 1 2 3 4        0 1 2 3 4
+        ┌─┬─┬─┬─┬─┐      ┌─┬─┬─┬─┬─┐
+      0 │1│ │ │ │8│    0 │1│ │ │ │8│
+        ├─┼─┼─┼─┼─┤      ├─┼─┼─┼─╱┬╲
+      1 │ │ │ │ │ │    1 │ │ │ │╱│││╲
+    R   ├─┼─┼─┼─┼─┤      ├─┼─┼─╱─┼▼┤ ╲
+    o 2 │ │ │3│ │ │    2 │ │ │3├─▶◉◀──?
+    w   ├─┼─┼─┼─┼─┤      ├─┼─┼─╲─┼▲┤ ╱
+      3 │ │ │ │ │ │    3 │ │ │ │╲│││╱
+        ├─┼─┼─┼─┼─┤      ├─┼─┼─┼─╲┴╱
+      4 │0│ │ │ │3│    4 │0│ │ │ │3│
+        └─┴─┴─┴─┴─┘      └─┴─┴─┴─┴─┘
+
+ +So we just ignore the missing point: + +```clojure +(defn safe-average [a b c d] + (let [total 0 count 0] + (when a (add! total a) (inc! count)) + (when b (add! total b) (inc! count)) + (when c (add! total c) (inc! count)) + (when d (add! total d) (inc! count)) + (/ total count))) + +(defn heightmap-get-safe [heightmap x y] + (let [last (heightmap-last-index heightmap)] + (when (and (<= 0 x last) + (<= 0 y last)) + (heightmap-get heightmap x y)))) +``` + +Technically this means they have less information to work with than the rest of +the points, and you might see some slight artifacts. But in practice it's not +too bad, and it's a lot nicer than midpoint displacement where *every* line step +uses only *two* points. + +Another way to handle this is to also wrap those edge coordinates around and +pull the point from the other side of the heightmap. This is a bit more work +but gives you an extra benefit: the heightmap will be tileable (**update**: like +everything where someone doesn't show you running code, it's [not actually that +simple](/blog/2016/08/lisp-jam-postmortem/#tiling-diamond-square)). + +## Iteration + +Unfortunately we can't use recursion to iterate for Diamond Square like we did +for Midpoint Displacement. We have to interleave the diamond and square steps, +performing each round of squares on the *entire* map before moving on to a round +of diamonds. If we don't, bad things will happen. + +Let's look at an example to see why. First we perform the initial square step: + +
+           Column
+         0 1 2 3 4        0 1 2 3 4
+        ┌─┬─┬─┬─┬─┐      ┌─┬─┬─┬─┬─┐
+      0 │1│ │ │ │8│    0 │1│ │ │ │8│
+        ├─┼─┼─┼─┼─┤      ├─╲─┼─┼─╱─┤
+      1 │ │ │ │ │ │    1 │ │╲│ │╱│ │
+    R   ├─┼─┼─┼─┼─┤      ├─┼─◢─◣─┼─┤
+    o 2 │ │ │ │ │ │    2 │ │ │3│ │ │
+    w   ├─┼─┼─┼─┼─┤      ├─┼─◥─◤─┼─┤
+      3 │ │ │ │ │ │    3 │ │╱│ │╲│ │
+        ├─┼─┼─┼─┼─┤      ├─╱─┼─┼─╲─┤
+      4 │0│ │ │ │3│    4 │0│ │ │ │3│
+        └─┴─┴─┴─┴─┘      └─┴─┴─┴─┴─┘
+
+ +That's fine. Then we recurse into the first of the four diamonds: + +
+           Column
+         0 1 2 3 4        0 1 2 3 4
+        ┌─┬─┬─┬─┬─┐      ┌─┬─┬─┬─┬─┐
+      0 │1│ │ │ │8│    0 │1├─▶4◀─┤8│
+        ├─┼─┼─┼─┼─┤      ├─╲─┼▲┼─╱─┤
+      1 │ │ │ │ │ │    1 │ │╲│││╱│ │
+    R   ├─┼─┼─┼─┼─┤      ├─┼─╲┴╱─┼─┤
+    o 2 │ │ │3│ │ │    2 │ │ │3│ │ │
+    w   ├─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┤
+      3 │ │ │ │ │ │    3 │ │ │ │ │ │
+        ├─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┤
+      4 │0│ │ │ │3│    4 │0│ │ │ │3│
+        └─┴─┴─┴─┴─┘      └─┴─┴─┴─┴─┘
+
+ +Everything's still good. Then we recurse into the square: + +
+            Column
+         0 1 2 3 4        0 1 2 3 4
+        ┌─┬─┬─┬─┬─┐      ╔═════╗─┬─┐
+      0 │1│ │4│ │8│    0 ║1│ │4║ │8│
+        ├─┼─┼─┼─┼─┤      ║─◢─◣─║─┼─┤
+      1 │ │ │ │ │ │    1 ║ │ │ ║ │ │
+    R   ├─┼─┼─┼─┼─┤      ║─◥─◤─║─┼─┤
+    o 2 │ │ │3│ │ │    2 ║?│ │3║ │ │
+    w   ├─┼─┼─┼─┼─┤      ╚═════╝─┼─┤
+      3 │ │ │ │ │ │    3 │ │ │ │ │ │
+        ├─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┤
+      4 │0│ │ │ │3│    4 │0│ │ │ │3│
+        └─┴─┴─┴─┴─┘      └─┴─┴─┴─┴─┘
+
+ +But wait, there's a corner missing! We needed to complete that left-hand +diamond step to get it, but that's still waiting on the stack for this line of +recursion to bottom out. + +### A "Strided" Iteration Construct + +So we need to use normal iteration. We can start by writing the code to do +a single round of diamonds or squares on the heightmap. First we'll make a nice +looping construct called `do-stride`: + +```clojure +(defmacro do-stride + [varnames start-form end-form stride-form & body] + (let [stride (gensym "stride") + start (gensym "start") + end (gensym "end") + build (fn build [vars] + (if (empty? vars) + `(do ~@body) + (let [varname (first vars)] + `(loop [~varname ~start] + (when (< ~varname ~end) + ~(build (rest vars)) + (recur (+ ~varname ~stride)))))))] + ; Fix the numbers once outside the nested loops, + ; and then build the guts. + `(let [~start ~start-form + ~end ~end-form + ~stride ~stride-form] + ~(build varnames)))) +``` + +This scary-looking macro builds the nested looping structure we'll need to +perform the iteration for our diamonds and squares. It abstracts away the +tedium of writing out nested `for` loops so we can focus on the rest of the +algorithm. + +`do-stride` takes a list of variables to bind, a number to start at, a number to +end before, and a body. For each variable it starts at `start`, runs `body`, +increments by `stride`, and loops until the value is at or above `end`. For +example: + +* `(do-stride [x] 0 5 1 (console.log x))` will log `0 1 2 3 4`. +* `(do-stride [x] 0 5 2 (console.log x))` will log `0 2 4`. +* `(do-stride [x y] 0 3 2 (console.log [x y]))` will log `[0, 0] [0, 2] [2, 0] + [2, 2]`. + +### Square Iteration + +Now that we've got this, we can iterate our square step over the heightmap for +a specific radius: + +```clojure +(defn ds-squares [heightmap radius spread] + (do-stride [x y] radius (heightmap-resolution heightmap) (* 2 radius) + (ds-square heightmap x y radius spread))) +``` + +We use `do-stride` to start at the radius and loop until we fall off the +heightmap, stepping by *twice* the radius each time. + +Because our heightmaps are always square we can use the same values for both +dimensions. `do-stride` can take zero or more variables and will just do the +right thing, so we don't need to worry about it. + +The stride is double the radius because as we step between squares we move over +the right half of the first *and* the left half of the next: + +
+               Column
+          0 1 2 3 4 5 6 7 8        0 1 2 3 4 5 6 7 8
+         ╔═════════╗─┬─┬─┬─┐      ╔═════════╗─┬─┬─┬─┐
+       0 ║1│ │ │ │3║ │ │ │3│    0 ║1│ │ │ │3║ │ │ │3│
+         ║─┼─┼─┼─┼─║─┼─┼─┼─┤      ║─┼─┼─┼─┼─║─┼─┼─┼─┤
+       1 ║ │ │ │ │ ║ │ │ │ │    1 ║ │ │ │ │ ║ │ │ │ │
+    R    ║─┼─┼─┼─┼─║─┼─┼─┼─┤     radius ┼─┼─║─┼─┼─┼─┤
+    o  2 ║ │ │◉│ │ ║ │◉│ │ │    2 ║◀━━▶◉◀━━━━━▶◉│ │ │
+    w    ║─┼─┼─┼─┼─║─┼─┼─┼─┤      ║─┼─┼─ stride ┼─┼─┤
+       3 ║ │ │ │ │ ║ │ │ │ │    3 ║ │ │ │ │ ║ │ │ │ │
+         ║─┼─┼─┼─┼─║─┼─┼─┼─┤      ║─┼─┼─┼─┼─║─┼─┼─┼─┤
+       4 ║5│ │ │ │5║ │ │ │5│    4 ║5│ │ │ │5║ │ │ │5│
+         ╚═════════╝─┼─┼─┼─┤      ╚═════════╝─┼─┼─┼─┤
+       5 │ │ │ │ │ │ │ │ │ │    5 │ │ │ │ │ │ │ │ │ │
+         ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
+       6 │ │ │◉│ │ │ │◉│ │ │    6 │ │ │◉│ │ │ │◉│ │ │
+         ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
+       7 │ │ │ │ │ │ │ │ │ │    7 │ │ │ │ │ │ │ │ │ │
+         ├─┼─┼─┼─┼─┼─┼─┼─┼─┤      ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
+       8 │9│ │ │ │7│ │ │ │7│    8 │9│ │ │ │7│ │ │ │7│
+         └─┴─┴─┴─┴─┴─┴─┴─┴─┘      └─┴─┴─┴─┴─┴─┴─┴─┴─┘
+
+ +### Diamond Iteration + +The diamonds are a little more complicated. + +First of all, in the `y` dimension we need to start at `0` instead of `radius` +(these will be the top and bottom edge cases we looked at). Also, in the `x` +direction all the even iterations need to start at `radius`, while the odd +iterations should start at `0`. + +Hopefully a picture will make it a bit more clear: + +
+               Column
+         0 1 2 3 4 5 6 7 8
+        ┌─┬─┬─┬─┬─┬─┬─┬─┬─┐
+      0 │1│ │◉│ │3│ │◉│ │3│ y iteration: 0, start at x = radius
+        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
+      1 │ │ │ │ │ │ │ │ │ │
+    R   ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
+    o 2 │◉│ │3│ │◉│ │4│ │◉│ y iteration: 1, start at x = 0
+    w   ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
+      3 │ │ │ │ │ │ │ │ │ │
+        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
+      4 │5│ │◉│ │5│ │◉│ │5│ y iteration: 2, start at x = radius
+        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
+      5 │ │ │ │ │ │ │ │ │ │
+        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
+      6 │◉│ │7│ │◉│ │6│ │◉│ y iteration: 3, start at x = 0
+        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
+      7 │ │ │ │ │ │ │ │ │ │
+        ├─┼─┼─┼─┼─┼─┼─┼─┼─┤
+      8 │9│ │◉│ │7│ │◉│ │7│ y iteration: 4, start at x = radius
+        └─┴─┴─┴─┴─┴─┴─┴─┴─┘
+
+ +The code for the diamonds is the hairiest bit of the algorithm. Make sure you +understand it before moving on. `shift` will be `radius` for even iterations +and `0` for odd ones: + +```clojure +(defn ds-diamonds [heightmap radius spread] + (let [size (heightmap-resolution heightmap)] + (do-stride [y] 0 size radius + (let [shift (if (even? (/ y radius)) radius 0)] + (do-stride [x] shift size (* 2 radius) + (ds-diamond heightmap x y radius spread)))))) +``` + +### Top-Level Iteration + +Finally we can put it all together. We initialize the corners and starting +values, then loop over smaller and smaller radii and just call `ds-squares` and +`ds-diamonds` to do the heavy lifting: + +```clojure +(defn diamond-square [heightmap] + (let [initial-spread 0.3 + spread-reduction 0.5 + center (heightmap-center-index heightmap) + size (aget heightmap.shape 0)] + (ds-init-corners heightmap) + (loop [radius center + spread initial-spread] + (when (>= radius 1) + (ds-squares heightmap radius spread) + (ds-diamonds heightmap radius spread) + (recur (/ radius 2) + (* spread spread-reduction)))) + (sanitize heightmap))) +``` + +## Result + +The end result looks like this: + +
+ +It looks a lot like the result from Midpoint Displacement, but without the ugly +seams around the square chunks. + +The code for these blog posts is a bit of a mess because I've been copy/pasting +to show the partially-completed demos after each step. I've created a little +[single-page demo][ymir] with completed versions of the various algorithms you +can play with, and [that code][ymir-code] should be a lot more readable than the +hacky code for these posts. + +[ymir]: http://ymir.stevelosh.com/ +[ymir-code]: http://bitbucket.org/sjl/ymir/ + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2016/06/symbolic-computation.html --- a/content/blog/2016/06/symbolic-computation.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,868 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "What the Hell is Symbolic Computation?" - snip: "Symbols, REPLs, and Quoting -- Oh My!" - created: 2016-06-29 13:30:00 - %} - - {% block article %} - -I've been reading a lot of Lisp books lately, some more advanced than others. -All of the introductory books I've seen cover the idea of symbolic computation, -but most of them breeze past it with just a few pages about "symbols" and -"quoting". For many programmers coming from non-Lisp languages these are new, -foreign concepts that don't really map back to anything in their previous -experience. But the books seem to expect you to understand this things after -just a couple of paragraphs. - -One book that *does* spend more time on this is the appropriately-named [Common -Lisp: A Gentle Introduction to Symbolic Computation][gentle]. If you're looking -for a good introductory Lisp book, that's the one I'd recommend. However, it's -quite a long book and starts at the *very* basics (you don't even write code for -the first few chapters -- you program with drawings). If you already know the -some Lisp then you might find yourself bored if you go through the entire thing -just for the symbolic computation bits. - -This post is an attempt to explain what symbols *actually are* and what quoting -does. You'll get the most out of it if you're already familiar with the basics -of Lisp (i.e. you aren't scared by the parentheses). But if you already know -what something like this would print, you'll probably be bored: - - :::text - (let ((foo ''a)) - (print foo) - (print 'foo) - (print (car foo)) - (print (cdr foo))) - -[gentle]: http://www.amazon.com/dp/0486498204/?tag=stelos-20 - -[TOC] - -## Disclaimer - -Before we start I'll get this warning out of the way: I'm going to gloss over -a lot of gory details to get at the important core ideas. - -When I say "this thing is implemented like so" it's safe to assume that in -reality things are messier and more complicated and will vary between -implementations and dialects. My goal is to give you some intuition for the -basic ideas, not to exhaustively cover how a particular implementation works. - -I'll be using Common Lisp for the examples, but the concepts apply equally to -Scheme, Clojure, and other Lisps too. - -With that out of the way, let's dive in! - -## The REPL - -The Read-Eval-Print-Loop that so many languages these days have built-in is -a good place to start in our exploration. You're probably familiar with the -idea of a REPL from languages like Python, Ruby, Javascript, etc. In Python it -looks like this: - - :::text - $ python - >>> if True: - ... print "Yes" - ... else: - ... print "No" - ... - Yes - -A handwavey definition of a REPL could be something like this: - -> A REPL is a program that lets you type in code. It runs that code, prints out -> the result, and loops back to the beginning to let you type in more. - -This is a good first start, but to understand symbols we're going to need to get -a much clearer, more precise handle on what exactly each letter in "REPL" means. -We'll start backwards and outside-in with the simplest component. - -## Loop - -The "L" in "REPL" stands for "Loop", and it's the easiest part to understand. -A REPL doesn't just process one bit of code and then exit, it loops forever (or -until you tell it to quit). - -There's not a lot to say here, except that we'll go ahead and lump together the -extra busywork a REPL needs to do into this step so we can ignore it for the -rest of the post. This includes things like: - -* Printing the nice prompt at the beginning (`>>>` or `...` in Python). -* Checking for `Ctrl-D` to quit. -* Handling exceptions (so they don't kill the process). - -## Print - -The next letter is "P" for "Print". Like `loop` this should be familiar -territory, but let's nail down *exactly* what it does. The job of the `print` -function is to: - -* Take as input some kind of object/data structure in memory. -* Produce some series of characters (a string) that represent this object. -* Write those characters somewhere (standard output, a text panel in a GUI, etc). - -For example: if we give the integer `1` to Python's `print`, it produces -a string of two characters (the digit `1` and a newline) and writes it to the -terminal. There are a lot of little details that aren't important here (line -endings, string encoding, where the output actually goes, etc). The important -part that you should fix in your mind is this: - -**`print` takes an object, converts it into a string, and outputs it.** - -Note: I've called `print` a function here, and I'll do the same for `read` and -`eval` later on. This is correct for Common Lisp, but not necessarily for -languages like Python or Javascript. For the purposes of this post you can just -imagine them as functions and realize that in the real world they're more like -"phases" that have a lot more complex machinery under them. - -## Read - -We'll jump over to the other side of the acronym now. "R" is for "Read", and -this is where things start to get tricky. - -In most non-Lisp languages the "read" and "eval" phases of the REPL tend to get -blurred together, but if you want to understand symbolic computation in Lisp you -absolutely *must* keep these two parts cleanly separated in your mind. - -We can define `read` as almost the polar opposite of `print`. `read` is -a function whose job is to: - -* Take as input a string of characters from some source. -* "Decode" that string and produce some kind of object in memory. -* Return that object. - -Notice how this mirrors `print`: - -* `print`: Object → Characters -* `read`: Characters → Object - -This is the second definition you need to fix in your brain: - -**`print` takes an object, converts it into a string, and outputs it.** -**`read` takes a string, converts it into an object, and returns it.** - -If we `read` in the string `1.0` (that's three ASCII characters), we'll get some -kind of floating point number object back. Sometimes there are many ways to -represent the same object in memory. For example: `read`ing the strings `1.0` -and `1.0000` will both produce equivalent floating point objects representing -the number "one". - -Side note (if you're not feeling pedantic feel free to skip this): the word -"equivalent" here is a bit tricky. When you `read` two strings like `1.0` and -`1.000` you might get back two pointers to the same hunk of memory, or you might -get back two pointers to two separate hunks of memory, or even just a machine -word representing the floating point number directly. It depends on the -language (and even the implementation). What I mean by "equivalent" is that -there's no way to tell which value came from which input. You can't say "this -hunk of memory must have come from the string `1.000`" because `read` has -"sanitized" the input by that point. - -## The RPL - -So now we've got `read`, `print`, and `loop`. `read` takes characters from -a stream and produces objects, and `print` takes objects and produces characters -and writes them to a stream. We're still missing a letter, but what would -happen if we just hook these three up together right now? - - :::text - CL-USER> (loop (print (read))) - > 1 - 1 - > 1.0 - 1.0 - > 1.00000 - 1.0 - -I've added a little `>` prompt to the output here to make it clear where I'm -typing. - -Essentially what we've got here is a little pretty-printer! Notice how things -like extra whitespace and useless zeroes got stripped out. This isn't terribly -useful, but at least it's doing *something*. - -However, it's important to realize where the "prettifying" actually happened: -`read` actually did the hard work! By the time `print` got its grubby little -paws on those one-point-zeroes the extra whitespace and digits were long gone. -Maybe we should give credit where it's due and call this a "pretty-reader" -instead. - -## Reading Other Data - -Up to this point I've been using numbers as examples because they're simple, but -of course we can read and print lots of other kinds of things. Lists and -strings (and lists *of* strings!) are some common examples: - - :::text - CL-USER> (print (read)) - > ("hello" "world") - ("hello" "world") - - $ python - >>> [ "hello" , "world"] - ['hello', 'world'] - -There are lots of other kinds of things you can read and print, depending on the -particular language. What's important here is the strict separation of `read` -and `print`. - -## Readable Printing - -Before we move on, one thing we should talk about is the idea of "readable -printing". This idea becomes clear almost immediately when you work at the -Python REPL: - - :::text - $ python - >>> "foo\nbar" - 'foo\nbar' - >>> print "foo\nbar" - foo - bar - -Why was the first string printed with quotes and an escaped newline, and the -second without quotes and an actual newline? - -When we defined `print` earlier, one of the steps was: - -* Produce some series of characters (a string) that represent this object. - -Much like there are many character strings that result in the same object (e.g. -`1.0` and `1.000`) there are often many ways we can represent a particular -object as a series of characters. For example: we can represent the number two -as `2`, or `10` in binary, or `two` in English. We can represent a string as -the actual characters in it, or we could represent it "escaped" like Python does -in the first line. - -The core idea here is that of **"readable printing": printing the characters you -would need to give to `read` to produce this object**. - -This is different from what I'll call **"aesthetic printing": printing -characters for this object which look nice to a human**. - -In Python these two kinds of printing are seen in the separate [`__str__` and -`__repr__` methods][str_repr]. In Common Lisp it's controlled by the -[`*print-readably*` variable][print-readably]. Other languages have different -mechanisms. - -[str_repr]: https://brennerm.github.io/posts/python-str-vs-repr.html -[print-readably]: http://clhs.lisp.se/Body/v_pr_rda.htm - -## Reading Non-Lisp - -So we know how reading and printing *data* works, but you can do more than that -at a typical REPL. Let's revisit the first Python example: - - :::text - $ python - >>> if True: - ... print "Yes" - ... else: - ... print "No" - ... - Yes - -We know that `read` takes a series of characters and turns them into an object. -When we `read` this series of characters, what comes out the other end? For -most languages (Python, Ruby, Javascript, Scala, etc) the answer is some kind of -[abstract syntax tree][] represented as a data structure. You might imagine -something like this buried deep within the guts of the Python interpreter: - - :::python - class IfStatement(ASTNode): - def __init__(self, test_node, then_node, else_node): - self.test_node = test_node - self.then_node = then_node - self.else_node = else_node - -The details would surely be a lot hairier in the real world, but the general -idea is that the stream of characters representing your code gets read in and -parsed into some kind of data structure in memory. - -This is all well and good, but our example printed "Yes", which certainly isn't -a representation of an AST node. Where did that come from? To answer that -we'll need the final letter in the acronym. - -[abstract syntax tree]: https://en.wikipedia.org/wiki/Abstract_syntax_tree - -## Evaluating Non-Lisp - -The "E" in REPL stands for "Eval". - -This is the last time we'll consider Python in this post, because we're about to -move past it to more powerful ideas. In Python, the purpose of the `eval` phase -of the REPL is to: - -* Take an AST node (or simple data) as input. -* Run the code (also known as *evaluating* it). -* Return the result (if any). - -This is fairly straightforward, at least for our simple `if` statement. Python -will evaluate the `test_node` part of the `if`, and then evaluate one of the -other parts depending on whether the test was truthy or falsey. - -Of course the actual process of evaluating real-life Python code is much more -complicated than I've glossed over here, but this is good enough for us to use -as a springboard to dive into the beautiful world of Lisp. - -## Reading Lisp - -When we talked about reading the Python `if` statement I said that Python would -parse it into some kind of data structure representing an AST node. I didn't -point you at the *actual* data structure in the Python source code because -I have no idea where it actually *is*, and it's probably not nearly as simple as -the one I sketched out. - -Now that we're talking about Lisp, however, I *really can* show you exactly what -an `if` statement in Lisp reads as! I'll use `read-from-string` to make it -a bit easier to see what's input and what's output, but it would work the same -way with just plain old `read` and input from standard in. Here we go: - - :::text - CL-USER> (print - (read-from-string "(if t (print 1) (print 2))")) - - (IF T (PRINT 1) (PRINT 2)) - - CL-USER> (type-of - (read-from-string "(if t (print 1) (print 2))")) - - CONS - -Instead of some kind of AST node data structure buried deep within the heart of -our implementation we get... a list! An ordinary, garden-variety, vanilla list -made of good old trusty cons cells! - -The list we get contains four elements, two of which are themselves lists. But -what do these lists actually *contain*? - -The numbers are simple enough -- we saw them earlier. `read` turns a series of -digits into a number somewhere in memory, and `print` spits out a series of -characters that represent the number: - - :::text - CL-USER> (type-of (read-from-string "100")) - - (INTEGER 0 4611686018427387903) - -The string `100` does indeed represent an integer between zero and four -squillion-something. - -But what are those *other* things in our list? The `if` and `t` and `print`? -Are they some magic keywords internal to the guts of our Lisp that `print` is -just outputting nicely for our benefit? - - :::text - CL-USER> (type-of (read-from-string "if")) - - SYMBOL - -Buckle up -- it's time to learn about symbols. - -## Symbols - -A symbol in Lisp is [a simple data structure][clhs-symbol] that turns out to be -*extremely* useful. You can imagine it being defined something like this: - - :::text - (defstruct symbol - name - function - value - property-list - package) - -The `name` of a symbol is a string of characters representing the symbol's name: - - :::text - CL-USER> (type-of (read-from-string "if")) - - SYMBOL - - CL-USER> (symbol-name (read-from-string "if")) - - "IF" - -Symbols get created (usually) when they are `read` for the first time. If you -`read` a sequence of characters that matches the name of an existing symbol -(let's ignore the lowercase/uppercase thing for now), `read` just returns that -existing symbol: - - :::text - CL-USER> (eq (read-from-string "cats") - (read-from-string "cats")) - - T - -Here `read` saw the string `cats` and created a new symbol somewhere in memory -with `CATS` as its `name`. The second call to `read-from-string` read in the -string `cats`, saw an existing symbol with the matching name, and just returned -it straight away. That's why `eq` returned true -- the two `read` calls -returned pointers to the exact same hunk of memory. - -(`read` also "interned" the symbol into the current package. We're going to -ignore the package system for this post -- it's not important to the core -topic. We'll also ignore the property list.) - -So to recap: - -* `read` is a function that takes characters and turns them into objects. -* When you feed `read` certain strings of characters (i.e. without special - characters), it returns a symbol object whose name matches those characters - (creating it if necessary). -* Symbols are just normal Lisp data structures with a few specific fields. - -So what does `print` do with a symbol? It just prints the `name`! - - :::text - CL-USER> (print (read-from-string "cats")) - - CATS - -Simple enough. Note that this is *readably printed* -- if we feed it back to -`read` we'll get an equivalent symbol back (the *exact same* symbol, in fact). - -But wait! If we `read` symbols in and they're just normal vanilla structures, -then how do we actually *do* anything in Lisp? How does anything actually -*run*? - -[clhs-symbol]: http://www.lispworks.com/documentation/lw70/CLHS/Body/t_symbol.htm - -## Evaluating Lisp - -Now we need to go back to `eval`. In the Python example I handwaved a bit and -said that `eval` "ran the code" but didn't really explain how that happened. -Now that we're in the warm embrace of Lisp we can nail things down a lot more -cleanly. - -`eval` (in Lisp) is a function whose job is to: - -* Take an object. -* Do something useful with it to get another object. -* Return the resulting object. - -I know, I know, that seems as clear as a brick. But if we can actually define -what "something useful" means it will be a lot more helpful. Lisp's `eval` -follows a few simple rules to produce its output. - -### Strings and Numbers - -First of all, `eval` will just pass numbers and strings straight through without -touching them at all: - - :::lisp - (defun eval (thing) - (cond - ((numberp thing) thing) - ((stringp thing) thing) - ; ... - (t (error "What the hell is this?")))) - -This makes sense because there's not a lot more useful stuff you can do with -a number or a string on its own. I suppose you could write a language where the -evaluation phase ran all numbers through the absolute value function because you -wanted to remove all negativity from programming or something, but that seems -a bit esoteric. - -### Basic Lists - -What about lists, like our friend the `if` statement (a list of four things)? -Well the first thing anyone learns about Lisp is the funny syntax. In most -cases, when passed a list `eval` treats the first element as the name of -a function, and the rest as the arguments. It will evaluate the arguments and -call the function with them: - - :::lisp - (defun eval (thing) - (cond - ((numberp thing) thing) - ((stringp thing) thing) - ((typep thing 'list) - (destructuring-bind (head . arguments) thing - (apply (symbol-function head) - (mapcar #'eval arguments)))) - ; ... - (t (error "What the hell is this?")))) - -At this point we know enough to understand exactly what happens when we say -something like `(+ 1 2)` at a Lisp REPL! - -* Lisp `read`s the seven characters we gave it, which becomes a list of three - objects: the symbol `+`, the integer `1`, and the integer `2`. -* Lisp passes that three-object list to `eval`. -* `eval` looks up what is in the `function` slot of the symbol at the front of - the list (that symbol is `+`). -* The symbol `+` was helpfully given a function by the Lisp implementation (run - `(symbol-function (read-from-string "+"))` to confirm that I'm not lying to - you). -* Lisp evaluates each item in the rest of the list. In this case they're each - a number, so each one evaluates to itself. -* Lisp calls the function with the evaluated arguments to get a result (in this - case: the integer `3`). -* Lisp passes the result to `print`, which turns it into a string of characters - (one character, in this case) and prints it to standard out. - -Whew! That seems like a lot, but each piece is pretty small. Go slowly through -it and make sure you *really* understand each step in the preceding list before -you move on. - -If you don't believe me that this is really how it works, try this: - - :::text - CL-USER> (setf (symbol-function (read-from-string "square")) - (lambda (x) (* x x))) - - # - - CL-USER> (square 4) - - 16 - -We set the `function` slot of the symbol `square` to be a function that squares -its argument, and then we can run it like any other function. - -Trace through the evaluation of that last `(square 4)` call before moving on. - -### Variables - -What about symbols that *aren't* at the front of a list? For example: - - :::text - CL-USER> (defvar ten 10) - - 10 - - CL-USER> (square ten) - - 100 - -In this case, everything proceeds as it did before until the recursive call to -`eval` the argument to `square`. The argument is just a bare symbol (not -a number or a list or a string), so we need another rule in our `eval` function: - - :::lisp - (defun eval (thing) - (cond - ((numberp thing) thing) - ((stringp thing) thing) - ((symbolp thing) (symbol-value thing)) ; NEW - ((listp thing) - (destructuring-bind (head . arguments) - (apply (symbol-function head) - (mapcar #'eval arguments)))) - ; ... - (t (error "What the hell is this?")))) - -If `eval` sees a symbol on its own, it will just return whatever is in the -`value` slot of the symbol data structure. Simple enough! - -### Special Forms -If course there are a few things we haven't talked about yet. First of all: -there are some "magic" symbols that aren't functions, but are hardcoded into the -evaluation function to do certain things. In Common Lisp they're called -["special operators"][special-operators] and there are twenty-five of them. - -Let's add a special rule to our `eval` function to handle one of them (our old -friend `if`): - - :::lisp - (defun eval (thing) - (cond - ((numberp thing) thing) - ((stringp thing) thing) - ((symbolp thing) (symbol-value thing)) - ((listp thing) - (destructuring-bind (head . arguments) thing - (cond - ((eq head (read-from-string "if")) ; NEW - (destructuring-bind (test then else) arguments - (if (eval test) - (eval then) - (eval else)))) - (t - (apply (symbol-function head) - (mapcar #'eval arguments)))))) - ; ... - (t (error "What the hell is this?")))) - -At this point your head might be starting to hurt. How have we just used `if` -to define itself? Is anything here actually *real*, or is Lisp just a serpent -eating its own tail forever? - -### An Ouroborromean Paradox? - -If the "metacircularity" (god, I hate that word) of what we've just done bothers -you, remember that this `eval` function here is just a sample of how you *can* -write it in Lisp. You could also write it in some other language, like C or -Java (see: [ABCL][]) or really anything. That would let you "bottom out" to -something that's not Lisp. - -But really, is that any *less* circular? You still need *some* language to run -the thing! Why does an `eval` written in C seem less "cheaty" than one written -in Lisp? Because it compiles to machine code? Try -`(disassemble #'sb-impl::simple-eval-in-lexenv)` in SBCL and you'll get a few -thousand bytes of x86 assembly that was [originally some Lisp code much like -ours][sbcl-eval]. - -My advice is to just let it flow over you, even if you feel a bit uneasy at the -sight of something like this. Eventually as you become more at home in your -chosen Lisp, writing high-level code one moment and `disassemble`ing a function -the next, you'll make peace with the serpent. - -### Exercises - -There are quite a few things in `eval` that we haven't covered here, like the -other twenty-four special operators, lexical variables, lambda forms, and more. -But this should be enough for you to wrap your head around symbols and how -they're used in Lisp. - -Before we move on to the final part of this story, make sure you can figure out -the answers to these questions: - -1. What does the single character `1` read as? What does it evaluate to? -2. What do the characters `"hello\"world"` read as? What do they evaluate to? - How would the result be printed readably? How would the result be printed - aesthetically? -3. What do the characters `(list (+ 1 2) ten)` read as? What does it evaluate - to? How does each step of the evaluation work? Trace out each of the calls - to `eval` on paper, with their arguments and their results. What does the - final result print as? -4. Use `symbol-function` to see what's in the function slot of the symbol `car`. - How does your Lisp print that thing? -5. Try to look up the function slot of a symbol without a function. What - happens? -4. Use `symbol-function` to see what's in the function slot of the symbol `if`. - What happens? Does that seem right? If so, why? -5. Go to the [HyperSpec page for `symbol`][clhs-symbol] and search for - "the consequences are undefined if". Try all of those things and find out - what interesting things break in your Lisp. - -If you're not using Common Lisp, port these exercises to your language. - -[special-operators]: http://www.lispworks.com/documentation/HyperSpec/Body/03_ababa.htm -[ABCL]: https://common-lisp.net/project/armedbear/ -[sbcl-eval]: https://github.com/sbcl/sbcl/blob/fdc4e9fa86b5eaaf8939f004a66e4be075069aa8/src/code/eval.lisp#L131-L272 - -## Quote - -Up to now we've been using the clumsy `(read-from-string "foo")` form to get -ahold of symbols, but there's an easier way called "quoting". - -Quoting often confuses new Lisp programmers, and they'll end up randomly -adding/removing quotes to something until it appears to do what they want. Now -that you've got `read`, `eval`, and `print` firmly separated in your mind, -`quote` will be much easier to understand. - -### The Special Operator - -Quoting actually has two distinct components: one that affects `read` and one -that affects `eval`. In most books the distinction between these two only gets -barely a sentence or two, which is probably why beginners are so confused. -Let's look at the `eval` side of things first. - -`quote` is one of the twenty-five special operators. All it does it pass its -(single) argument back from `eval` *untouched*. Normally the argument itself -would have been evaluated, but `quote` prevents that: - - :::lisp - (defun eval (thing) - (cond - ((numberp thing) thing) - ((stringp thing) thing) - ((symbolp thing) (symbol-value thing)) - ((listp thing) - (destructuring-bind (head . arguments) thing - (cond - ((eq head (read-from-string "if")) - (destructuring-bind (test then else) arguments - (if (eval test) - (eval then) - (eval else)))) - ((eq head (read-from-string "quote")) ; NEW - (first arguments)) - (t - (apply (symbol-function head) - (mapcar #'eval arguments)))))) - ; ... - (t (error "What the hell is this?")))) - -I know that seems strange, but let's look at it in action: - - :::text - CL-USER> (+ 1 2) - - 3 - - CL-USER> (quote (+ 1 2)) - - (+ 1 2) - -Our first example is the same one we used in the previous section: - -* `read` takes in seven characters and returns a list of three items: the symbol - `+` and two numbers. -* `eval` gets the three-element list, looks up the function for the symbol `+`, - recursively evaluates the arguments (which evaluate to themselves), and - applies the function. -* `print` gets the integer `3`, turns it into ASCII, and writes it to the - terminal. - -The second example uses `quote`: - -* `read` takes in fifteen characters and returns a list of two items: the symbol - `quote` and a list of three items (the symbol `+` and two numbers). -* `eval` gets the three element list and notices that the first element is the - special operator `quote`. It simply returns the second element in the list - untouched, which in this case is a list of three items. -* `print` gets that three element list, turns it into seven characters, and - writes it to the terminal. - -Just for fun, let's ride the serpent and use `quote` in the definition of -itself inside `eval` to clean things up a bit: - - :::lisp - (defun eval (thing) - (cond - ((numberp thing) thing) - ((stringp thing) thing) - ((symbolp thing) (symbol-value thing)) - ((listp thing) - (destructuring-bind (head . arguments) thing - (cond - ((eq head (quote if)) - (destructuring-bind (test then else) arguments - (if (eval test) - (eval then) - (eval else)))) - ((eq head (quote quote)) ; WHAT IN THE - (first arguments)) - (t - (apply (symbol-function head) - (mapcar #'eval arguments)))))) - ; ... - (t (error "What the hell is this?")))) - -That `(quote quote)` looks just completely bonkers, right? Step through it -piece by piece to figure it out: - -* What do the five characters `quote` read as? What do they evaluate to? -* What do the thirteen (spooky!) characters `(quote quote)` read as? When - `eval` gets its hands on that, what clause in the `cond` does it hit? What - does that return? - -Try these in a Lisp REPL if you're not 100% sure of the answers. The functions -`read-from-string`, `eval`, and `type-of` will come in handy. - -### The Read Macro - -Now let's look at the final piece of the puzzle. `'` (that's a single ASCII -quote character) is a [read macro][]. Don't worry if you don't know exactly -what that means right now. The important part is that when `read` sees a quote -character, it returns a two-element list containing the symbol `quote` and the -next form, which is read normally. - -The four characters `'foo` go into `read` and come out as a two-element list -containing: the symbol `quote` and the symbol `foo`. Again, because it's -important: this happens *at read time*. By the time `eval` sees anything it's -already been turned into that two-element list. - -It can be a bit slippery to get a grip on this, because `print` will "helpfully" -print a list of `(quote whatever)` as `'whatever`! But we can use trusty old -`car` and `cdr` to see what's really going on: - - :::lisp - CL-USER> (read-from-string "'foo") - - 'FOO - - CL-USER> (type-of (read-from-string "'foo")) - - CONS - - CL-USER> (car (read-from-string "'foo")) - - QUOTE - - CL-USER> (cdr (read-from-string "'foo")) - - (FOO) - -That's really all there is to it: `'form` is returned by `read` as `(quote -form)`. Then that gets passed along to `eval`, which sees the `quote` special -operator and passes the argument back untouched. - -Note that `form` doesn't have to be a symbol: - -* `'(1 2 (4 5))` reads as `(quote (1 2 (4 5)))` -* `'150` reads as `(quote 150)` -* `'"hello"` reads as `(quote "hello")` -* `'(if if if if)` reads as `(quote (if if if if))` - -[read macro]: https://gist.github.com/chaitanyagupta/9324402 - -### Exercises - -That pretty much wraps up `quote`. Once you've cleanly separated `read`, -`eval`, and `print` in your mind it becomes a lot less mystical (or perhaps -*more*, depending on your point of view). - -If you want to twist your brain just a bit more, try these exercises: - -1. What do the characters `foo` read in as? What do they evaluate to? How does it print? -2. What do the characters `'foo` read in as? What do they evaluate to? How does it print? -3. What do the characters `''foo` read in as? What do they evaluate to? How does it print? -4. What do the characters `'1.0` read in as? What do they evaluate to? How does it print? -5. What do the characters `'(1 2 3)` read in as? How many elements are in the list? What does it evaluate to? How many elements are in *that* list? -6. Update our implementation of `eval` to use the `'` read macro instead of `quote`. -7. Certain symbols like `t` and `nil` are special -- they evaluate to themselves like numbers or strings. Add this to `eval`. -8. What do the characters `nil` read in as? What is the type of that object? What do they evaluate to? What type is that result? -9. What do the characters `'nil` read in as? What is the type of that object? What do they evaluate to? What type is that result? -10. What do the characters `''nil` read in as? What is the type of that object? What do they evaluate to? What type is that result? - -## A Quick Recap - -If you've gotten this far and understood everything, you should have a good -grasp of how symbolic computation works. Here are the main things I hope you've -taken away from this post: - -* `read`, `eval`, and `print` are three separate, distinct functions/phases in - the interpretation of Lisp (and really *all* (okay, *most*)) code. -* Keeping these three phases *clearly separated* in your mind will make it - easier to understand symbols and quoting in Lisp. -* Symbols are nothing magic, they're just a particular data structure with - a couple of fields. -* The important thing is how `read` and `eval` *treat* symbols. That's where - the magic happens. -* `quote` is just a "short-circuit" in `eval` that turns out to be really - useful. -* `'` is just a lazier way of writing `(quote ...)`. - -## Where to Go From Here - -Now that you've got a firmer grasp on what the hell symbols actually *are*, -there are a lot of cool things you might want to check out that will show you -what you can do with them: - -* Reread some of your introductory Lisp books again, seeing if things seem to - make a bit more sense now. -* Read about backquote/quasiquote. How does it compare to the `quote` we - explored here? -* Read [Paradigms of Artificial Intelligence Programming][PAIP] and pay - attention to how Norvig uses symbols for various tasks. Also pay attention to - what he says about property lists, because we skipped them here. -* Read the [guide to the Common Lisp package system][packaging] to learn all the - stuff I left out about the Common Lisp package system. -* Gaze upon [this macro][symbolicate] in wonder (or horror). -* Read the HyperSpec pages for things like `symbol` and `quote`. -* Find and read your Lisp implementation's `eval` function. -* Find and read *another* Lisp implementation's `eval` function. -* Watch the [SICP Lectures][SICP] on YouTube. - -[PAIP]: http://www.amazon.com/dp/1558601910/?tag=stelos-20 -[symbolicate]: https://github.com/zkat/squirl/blob/f0dc57dfee728df94b2a35956e18e143e8b8d275/src/vec.lisp#L28-L38 -[packaging]: https://www-fourier.ujf-grenoble.fr/~sergerar/Papers/Packaging.pdf -[SICP]: http://www.amazon.com/dp/0262510871/?tag=stelos-20 - - {% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2016/06/symbolic-computation.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2016/06/symbolic-computation.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,889 @@ ++++ +title = "What the Hell is Symbolic Computation?" +snip = "Symbols, REPLs, and Quoting — Oh My!" +date = 2016-06-29T13:30:00Z +draft = false + ++++ + +I've been reading a lot of Lisp books lately, some more advanced than others. +All of the introductory books I've seen cover the idea of symbolic computation, +but most of them breeze past it with just a few pages about "symbols" and +"quoting". For many programmers coming from non-Lisp languages these are new, +foreign concepts that don't really map back to anything in their previous +experience. But the books seem to expect you to understand this things after +just a couple of paragraphs. + +One book that *does* spend more time on this is the appropriately-named [Common +Lisp: A Gentle Introduction to Symbolic Computation][gentle]. If you're looking +for a good introductory Lisp book, that's the one I'd recommend. However, it's +quite a long book and starts at the *very* basics (you don't even write code for +the first few chapters — you program with drawings). If you already know the +some Lisp then you might find yourself bored if you go through the entire thing +just for the symbolic computation bits. + +This post is an attempt to explain what symbols *actually are* and what quoting +does. You'll get the most out of it if you're already familiar with the basics +of Lisp (i.e. you aren't scared by the parentheses). But if you already know +what something like this would print, you'll probably be bored: + +```text +(let ((foo ''a)) + (print foo) + (print 'foo) + (print (car foo)) + (print (cdr foo))) +``` + +[gentle]: http://www.amazon.com/dp/0486498204/?tag=stelos-20 + +{{% toc %}} + +## Disclaimer + +Before we start I'll get this warning out of the way: I'm going to gloss over +a lot of gory details to get at the important core ideas. + +When I say "this thing is implemented like so" it's safe to assume that in +reality things are messier and more complicated and will vary between +implementations and dialects. My goal is to give you some intuition for the +basic ideas, not to exhaustively cover how a particular implementation works. + +I'll be using Common Lisp for the examples, but the concepts apply equally to +Scheme, Clojure, and other Lisps too. + +With that out of the way, let's dive in! + +## The REPL + +The Read-Eval-Print-Loop that so many languages these days have built-in is +a good place to start in our exploration. You're probably familiar with the +idea of a REPL from languages like Python, Ruby, Javascript, etc. In Python it +looks like this: + +```text +$ python +>>> if True: +... print "Yes" +... else: +... print "No" +... +Yes +``` + +A handwavey definition of a REPL could be something like this: + +> A REPL is a program that lets you type in code. It runs that code, prints out +> the result, and loops back to the beginning to let you type in more. + +This is a good first start, but to understand symbols we're going to need to get +a much clearer, more precise handle on what exactly each letter in "REPL" means. +We'll start backwards and outside-in with the simplest component. + +## Loop + +The "L" in "REPL" stands for "Loop", and it's the easiest part to understand. +A REPL doesn't just process one bit of code and then exit, it loops forever (or +until you tell it to quit). + +There's not a lot to say here, except that we'll go ahead and lump together the +extra busywork a REPL needs to do into this step so we can ignore it for the +rest of the post. This includes things like: + +* Printing the nice prompt at the beginning (`>>>` or `...` in Python). +* Checking for `Ctrl-D` to quit. +* Handling exceptions (so they don't kill the process). + +## Print + +The next letter is "P" for "Print". Like `loop` this should be familiar +territory, but let's nail down *exactly* what it does. The job of the `print` +function is to: + +* Take as input some kind of object/data structure in memory. +* Produce some series of characters (a string) that represent this object. +* Write those characters somewhere (standard output, a text panel in a GUI, etc). + +For example: if we give the integer `1` to Python's `print`, it produces +a string of two characters (the digit `1` and a newline) and writes it to the +terminal. There are a lot of little details that aren't important here (line +endings, string encoding, where the output actually goes, etc). The important +part that you should fix in your mind is this: + +**`print` takes an object, converts it into a string, and outputs it.** + +Note: I've called `print` a function here, and I'll do the same for `read` and +`eval` later on. This is correct for Common Lisp, but not necessarily for +languages like Python or Javascript. For the purposes of this post you can just +imagine them as functions and realize that in the real world they're more like +"phases" that have a lot more complex machinery under them. + +## Read + +We'll jump over to the other side of the acronym now. "R" is for "Read", and +this is where things start to get tricky. + +In most non-Lisp languages the "read" and "eval" phases of the REPL tend to get +blurred together, but if you want to understand symbolic computation in Lisp you +absolutely *must* keep these two parts cleanly separated in your mind. + +We can define `read` as almost the polar opposite of `print`. `read` is +a function whose job is to: + +* Take as input a string of characters from some source. +* "Decode" that string and produce some kind of object in memory. +* Return that object. + +Notice how this mirrors `print`: + +* `print`: Object → Characters +* `read`: Characters → Object + +This is the second definition you need to fix in your brain: + +**`print` takes an object, converts it into a string, and outputs it.** +**`read` takes a string, converts it into an object, and returns it.** + +If we `read` in the string `1.0` (that's three ASCII characters), we'll get some +kind of floating point number object back. Sometimes there are many ways to +represent the same object in memory. For example: `read`ing the strings `1.0` +and `1.0000` will both produce equivalent floating point objects representing +the number "one". + +Side note (if you're not feeling pedantic feel free to skip this): the word +"equivalent" here is a bit tricky. When you `read` two strings like `1.0` and +`1.000` you might get back two pointers to the same hunk of memory, or you might +get back two pointers to two separate hunks of memory, or even just a machine +word representing the floating point number directly. It depends on the +language (and even the implementation). What I mean by "equivalent" is that +there's no way to tell which value came from which input. You can't say "this +hunk of memory must have come from the string `1.000`" because `read` has +"sanitized" the input by that point. + +## The RPL + +So now we've got `read`, `print`, and `loop`. `read` takes characters from +a stream and produces objects, and `print` takes objects and produces characters +and writes them to a stream. We're still missing a letter, but what would +happen if we just hook these three up together right now? + +```text +CL-USER> (loop (print (read))) +> 1 +1 +> 1.0 +1.0 +> 1.00000 +1.0 +``` + +I've added a little `>` prompt to the output here to make it clear where I'm +typing. + +Essentially what we've got here is a little pretty-printer! Notice how things +like extra whitespace and useless zeroes got stripped out. This isn't terribly +useful, but at least it's doing *something*. + +However, it's important to realize where the "prettifying" actually happened: +`read` actually did the hard work! By the time `print` got its grubby little +paws on those one-point-zeroes the extra whitespace and digits were long gone. +Maybe we should give credit where it's due and call this a "pretty-reader" +instead. + +## Reading Other Data + +Up to this point I've been using numbers as examples because they're simple, but +of course we can read and print lots of other kinds of things. Lists and +strings (and lists *of* strings!) are some common examples: + +```text +CL-USER> (print (read)) +> ("hello" "world") +("hello" "world") + +$ python +>>> [ "hello" , "world"] +['hello', 'world'] +``` + +There are lots of other kinds of things you can read and print, depending on the +particular language. What's important here is the strict separation of `read` +and `print`. + +## Readable Printing + +Before we move on, one thing we should talk about is the idea of "readable +printing". This idea becomes clear almost immediately when you work at the +Python REPL: + +```text +$ python +>>> "foo\nbar" +'foo\nbar' +>>> print "foo\nbar" +foo +bar +``` + +Why was the first string printed with quotes and an escaped newline, and the +second without quotes and an actual newline? + +When we defined `print` earlier, one of the steps was: + +* Produce some series of characters (a string) that represent this object. + +Much like there are many character strings that result in the same object (e.g. +`1.0` and `1.000`) there are often many ways we can represent a particular +object as a series of characters. For example: we can represent the number two +as `2`, or `10` in binary, or `two` in English. We can represent a string as +the actual characters in it, or we could represent it "escaped" like Python does +in the first line. + +The core idea here is that of **"readable printing": printing the characters you +would need to give to `read` to produce this object**. + +This is different from what I'll call **"aesthetic printing": printing +characters for this object which look nice to a human**. + +In Python these two kinds of printing are seen in the separate [`__str__` and +`__repr__` methods][str_repr]. In Common Lisp it's controlled by the +[`*print-readably*` variable][print-readably]. Other languages have different +mechanisms. + +[str_repr]: https://brennerm.github.io/posts/python-str-vs-repr.html +[print-readably]: http://clhs.lisp.se/Body/v_pr_rda.htm + +## Reading Non-Lisp + +So we know how reading and printing *data* works, but you can do more than that +at a typical REPL. Let's revisit the first Python example: + +```text +$ python +>>> if True: +... print "Yes" +... else: +... print "No" +... +Yes +``` + +We know that `read` takes a series of characters and turns them into an object. +When we `read` this series of characters, what comes out the other end? For +most languages (Python, Ruby, Javascript, Scala, etc) the answer is some kind of +[abstract syntax tree][] represented as a data structure. You might imagine +something like this buried deep within the guts of the Python interpreter: + +```python +class IfStatement(ASTNode): + def __init__(self, test_node, then_node, else_node): + self.test_node = test_node + self.then_node = then_node + self.else_node = else_node +``` + +The details would surely be a lot hairier in the real world, but the general +idea is that the stream of characters representing your code gets read in and +parsed into some kind of data structure in memory. + +This is all well and good, but our example printed "Yes", which certainly isn't +a representation of an AST node. Where did that come from? To answer that +we'll need the final letter in the acronym. + +[abstract syntax tree]: https://en.wikipedia.org/wiki/Abstract_syntax_tree + +## Evaluating Non-Lisp + +The "E" in REPL stands for "Eval". + +This is the last time we'll consider Python in this post, because we're about to +move past it to more powerful ideas. In Python, the purpose of the `eval` phase +of the REPL is to: + +* Take an AST node (or simple data) as input. +* Run the code (also known as *evaluating* it). +* Return the result (if any). + +This is fairly straightforward, at least for our simple `if` statement. Python +will evaluate the `test_node` part of the `if`, and then evaluate one of the +other parts depending on whether the test was truthy or falsey. + +Of course the actual process of evaluating real-life Python code is much more +complicated than I've glossed over here, but this is good enough for us to use +as a springboard to dive into the beautiful world of Lisp. + +## Reading Lisp + +When we talked about reading the Python `if` statement I said that Python would +parse it into some kind of data structure representing an AST node. I didn't +point you at the *actual* data structure in the Python source code because +I have no idea where it actually *is*, and it's probably not nearly as simple as +the one I sketched out. + +Now that we're talking about Lisp, however, I *really can* show you exactly what +an `if` statement in Lisp reads as! I'll use `read-from-string` to make it +a bit easier to see what's input and what's output, but it would work the same +way with just plain old `read` and input from standard in. Here we go: + +```text +CL-USER> (print + (read-from-string "(if t (print 1) (print 2))")) + +(IF T (PRINT 1) (PRINT 2)) + +CL-USER> (type-of + (read-from-string "(if t (print 1) (print 2))")) + +CONS +``` + +Instead of some kind of AST node data structure buried deep within the heart of +our implementation we get... a list! An ordinary, garden-variety, vanilla list +made of good old trusty cons cells! + +The list we get contains four elements, two of which are themselves lists. But +what do these lists actually *contain*? + +The numbers are simple enough — we saw them earlier. `read` turns a series of +digits into a number somewhere in memory, and `print` spits out a series of +characters that represent the number: + +```text +CL-USER> (type-of (read-from-string "100")) + +(INTEGER 0 4611686018427387903) +``` + +The string `100` does indeed represent an integer between zero and four +squillion-something. + +But what are those *other* things in our list? The `if` and `t` and `print`? +Are they some magic keywords internal to the guts of our Lisp that `print` is +just outputting nicely for our benefit? + +```text +CL-USER> (type-of (read-from-string "if")) + +SYMBOL +``` + +Buckle up — it's time to learn about symbols. + +## Symbols + +A symbol in Lisp is [a simple data structure][clhs-symbol] that turns out to be +*extremely* useful. You can imagine it being defined something like this: + +```text +(defstruct symbol + name + function + value + property-list + package) +``` + +The `name` of a symbol is a string of characters representing the symbol's name: + +```text +CL-USER> (type-of (read-from-string "if")) + +SYMBOL + +CL-USER> (symbol-name (read-from-string "if")) + +"IF" +``` + +Symbols get created (usually) when they are `read` for the first time. If you +`read` a sequence of characters that matches the name of an existing symbol +(let's ignore the lowercase/uppercase thing for now), `read` just returns that +existing symbol: + +```text +CL-USER> (eq (read-from-string "cats") + (read-from-string "cats")) + +T +``` + +Here `read` saw the string `cats` and created a new symbol somewhere in memory +with `CATS` as its `name`. The second call to `read-from-string` read in the +string `cats`, saw an existing symbol with the matching name, and just returned +it straight away. That's why `eq` returned true — the two `read` calls +returned pointers to the exact same hunk of memory. + +(`read` also "interned" the symbol into the current package. We're going to +ignore the package system for this post — it's not important to the core +topic. We'll also ignore the property list.) + +So to recap: + +* `read` is a function that takes characters and turns them into objects. +* When you feed `read` certain strings of characters (i.e. without special + characters), it returns a symbol object whose name matches those characters + (creating it if necessary). +* Symbols are just normal Lisp data structures with a few specific fields. + +So what does `print` do with a symbol? It just prints the `name`! + +```text +CL-USER> (print (read-from-string "cats")) + +CATS +``` + +Simple enough. Note that this is *readably printed* — if we feed it back to +`read` we'll get an equivalent symbol back (the *exact same* symbol, in fact). + +But wait! If we `read` symbols in and they're just normal vanilla structures, +then how do we actually *do* anything in Lisp? How does anything actually +*run*? + +[clhs-symbol]: http://www.lispworks.com/documentation/lw70/CLHS/Body/t_symbol.htm + +## Evaluating Lisp + +Now we need to go back to `eval`. In the Python example I handwaved a bit and +said that `eval` "ran the code" but didn't really explain how that happened. +Now that we're in the warm embrace of Lisp we can nail things down a lot more +cleanly. + +`eval` (in Lisp) is a function whose job is to: + +* Take an object. +* Do something useful with it to get another object. +* Return the resulting object. + +I know, I know, that seems as clear as a brick. But if we can actually define +what "something useful" means it will be a lot more helpful. Lisp's `eval` +follows a few simple rules to produce its output. + +### Strings and Numbers + +First of all, `eval` will just pass numbers and strings straight through without +touching them at all: + +```lisp +(defun eval (thing) + (cond + ((numberp thing) thing) + ((stringp thing) thing) + ; ... + (t (error "What the hell is this?")))) +``` + +This makes sense because there's not a lot more useful stuff you can do with +a number or a string on its own. I suppose you could write a language where the +evaluation phase ran all numbers through the absolute value function because you +wanted to remove all negativity from programming or something, but that seems +a bit esoteric. + +### Basic Lists + +What about lists, like our friend the `if` statement (a list of four things)? +Well the first thing anyone learns about Lisp is the funny syntax. In most +cases, when passed a list `eval` treats the first element as the name of +a function, and the rest as the arguments. It will evaluate the arguments and +call the function with them: + +```lisp +(defun eval (thing) + (cond + ((numberp thing) thing) + ((stringp thing) thing) + ((typep thing 'list) + (destructuring-bind (head . arguments) thing + (apply (symbol-function head) + (mapcar #'eval arguments)))) + ; ... + (t (error "What the hell is this?")))) +``` + +At this point we know enough to understand exactly what happens when we say +something like `(+ 1 2)` at a Lisp REPL! + +* Lisp `read`s the seven characters we gave it, which becomes a list of three + objects: the symbol `+`, the integer `1`, and the integer `2`. +* Lisp passes that three-object list to `eval`. +* `eval` looks up what is in the `function` slot of the symbol at the front of + the list (that symbol is `+`). +* The symbol `+` was helpfully given a function by the Lisp implementation (run + `(symbol-function (read-from-string "+"))` to confirm that I'm not lying to + you). +* Lisp evaluates each item in the rest of the list. In this case they're each + a number, so each one evaluates to itself. +* Lisp calls the function with the evaluated arguments to get a result (in this + case: the integer `3`). +* Lisp passes the result to `print`, which turns it into a string of characters + (one character, in this case) and prints it to standard out. + +Whew! That seems like a lot, but each piece is pretty small. Go slowly through +it and make sure you *really* understand each step in the preceding list before +you move on. + +If you don't believe me that this is really how it works, try this: + +```text +CL-USER> (setf (symbol-function (read-from-string "square")) + (lambda (x) (* x x))) + +# + +CL-USER> (square 4) + +16 +``` + +We set the `function` slot of the symbol `square` to be a function that squares +its argument, and then we can run it like any other function. + +Trace through the evaluation of that last `(square 4)` call before moving on. + +### Variables + +What about symbols that *aren't* at the front of a list? For example: + +```text +CL-USER> (defvar ten 10) + +10 + +CL-USER> (square ten) + +100 +``` + +In this case, everything proceeds as it did before until the recursive call to +`eval` the argument to `square`. The argument is just a bare symbol (not +a number or a list or a string), so we need another rule in our `eval` function: + +```lisp +(defun eval (thing) + (cond + ((numberp thing) thing) + ((stringp thing) thing) + ((symbolp thing) (symbol-value thing)) ; NEW + ((listp thing) + (destructuring-bind (head . arguments) + (apply (symbol-function head) + (mapcar #'eval arguments)))) + ; ... + (t (error "What the hell is this?")))) +``` + +If `eval` sees a symbol on its own, it will just return whatever is in the +`value` slot of the symbol data structure. Simple enough! + +### Special Forms +If course there are a few things we haven't talked about yet. First of all: +there are some "magic" symbols that aren't functions, but are hardcoded into the +evaluation function to do certain things. In Common Lisp they're called +["special operators"][special-operators] and there are twenty-five of them. + +Let's add a special rule to our `eval` function to handle one of them (our old +friend `if`): + +```lisp +(defun eval (thing) + (cond + ((numberp thing) thing) + ((stringp thing) thing) + ((symbolp thing) (symbol-value thing)) + ((listp thing) + (destructuring-bind (head . arguments) thing + (cond + ((eq head (read-from-string "if")) ; NEW + (destructuring-bind (test then else) arguments + (if (eval test) + (eval then) + (eval else)))) + (t + (apply (symbol-function head) + (mapcar #'eval arguments)))))) + ; ... + (t (error "What the hell is this?")))) +``` + +At this point your head might be starting to hurt. How have we just used `if` +to define itself? Is anything here actually *real*, or is Lisp just a serpent +eating its own tail forever? + +### An Ouroborromean Paradox? + +If the "metacircularity" (god, I hate that word) of what we've just done bothers +you, remember that this `eval` function here is just a sample of how you *can* +write it in Lisp. You could also write it in some other language, like C or +Java (see: [ABCL][]) or really anything. That would let you "bottom out" to +something that's not Lisp. + +But really, is that any *less* circular? You still need *some* language to run +the thing! Why does an `eval` written in C seem less "cheaty" than one written +in Lisp? Because it compiles to machine code? Try +`(disassemble #'sb-impl::simple-eval-in-lexenv)` in SBCL and you'll get a few +thousand bytes of x86 assembly that was [originally some Lisp code much like +ours][sbcl-eval]. + +My advice is to just let it flow over you, even if you feel a bit uneasy at the +sight of something like this. Eventually as you become more at home in your +chosen Lisp, writing high-level code one moment and `disassemble`ing a function +the next, you'll make peace with the serpent. + +### Exercises + +There are quite a few things in `eval` that we haven't covered here, like the +other twenty-four special operators, lexical variables, lambda forms, and more. +But this should be enough for you to wrap your head around symbols and how +they're used in Lisp. + +Before we move on to the final part of this story, make sure you can figure out +the answers to these questions: + +1. What does the single character `1` read as? What does it evaluate to? +2. What do the characters `"hello\"world"` read as? What do they evaluate to? + How would the result be printed readably? How would the result be printed + aesthetically? +3. What do the characters `(list (+ 1 2) ten)` read as? What does it evaluate + to? How does each step of the evaluation work? Trace out each of the calls + to `eval` on paper, with their arguments and their results. What does the + final result print as? +4. Use `symbol-function` to see what's in the function slot of the symbol `car`. + How does your Lisp print that thing? +5. Try to look up the function slot of a symbol without a function. What + happens? +4. Use `symbol-function` to see what's in the function slot of the symbol `if`. + What happens? Does that seem right? If so, why? +5. Go to the [HyperSpec page for `symbol`][clhs-symbol] and search for + "the consequences are undefined if". Try all of those things and find out + what interesting things break in your Lisp. + +If you're not using Common Lisp, port these exercises to your language. + +[special-operators]: http://www.lispworks.com/documentation/HyperSpec/Body/03_ababa.htm +[ABCL]: https://common-lisp.net/project/armedbear/ +[sbcl-eval]: https://github.com/sbcl/sbcl/blob/fdc4e9fa86b5eaaf8939f004a66e4be075069aa8/src/code/eval.lisp#L131-L272 + +## Quote + +Up to now we've been using the clumsy `(read-from-string "foo")` form to get +ahold of symbols, but there's an easier way called "quoting". + +Quoting often confuses new Lisp programmers, and they'll end up randomly +adding/removing quotes to something until it appears to do what they want. Now +that you've got `read`, `eval`, and `print` firmly separated in your mind, +`quote` will be much easier to understand. + +### The Special Operator + +Quoting actually has two distinct components: one that affects `read` and one +that affects `eval`. In most books the distinction between these two only gets +barely a sentence or two, which is probably why beginners are so confused. +Let's look at the `eval` side of things first. + +`quote` is one of the twenty-five special operators. All it does it pass its +(single) argument back from `eval` *untouched*. Normally the argument itself +would have been evaluated, but `quote` prevents that: + +```lisp +(defun eval (thing) + (cond + ((numberp thing) thing) + ((stringp thing) thing) + ((symbolp thing) (symbol-value thing)) + ((listp thing) + (destructuring-bind (head . arguments) thing + (cond + ((eq head (read-from-string "if")) + (destructuring-bind (test then else) arguments + (if (eval test) + (eval then) + (eval else)))) + ((eq head (read-from-string "quote")) ; NEW + (first arguments)) + (t + (apply (symbol-function head) + (mapcar #'eval arguments)))))) + ; ... + (t (error "What the hell is this?")))) +``` + +I know that seems strange, but let's look at it in action: + +```text +CL-USER> (+ 1 2) + +3 + +CL-USER> (quote (+ 1 2)) + +(+ 1 2) +``` + +Our first example is the same one we used in the previous section: + +* `read` takes in seven characters and returns a list of three items: the symbol + `+` and two numbers. +* `eval` gets the three-element list, looks up the function for the symbol `+`, + recursively evaluates the arguments (which evaluate to themselves), and + applies the function. +* `print` gets the integer `3`, turns it into ASCII, and writes it to the + terminal. + +The second example uses `quote`: + +* `read` takes in fifteen characters and returns a list of two items: the symbol + `quote` and a list of three items (the symbol `+` and two numbers). +* `eval` gets the three element list and notices that the first element is the + special operator `quote`. It simply returns the second element in the list + untouched, which in this case is a list of three items. +* `print` gets that three element list, turns it into seven characters, and + writes it to the terminal. + +Just for fun, let's ride the serpent and use `quote` in the definition of +itself inside `eval` to clean things up a bit: + +```lisp +(defun eval (thing) + (cond + ((numberp thing) thing) + ((stringp thing) thing) + ((symbolp thing) (symbol-value thing)) + ((listp thing) + (destructuring-bind (head . arguments) thing + (cond + ((eq head (quote if)) + (destructuring-bind (test then else) arguments + (if (eval test) + (eval then) + (eval else)))) + ((eq head (quote quote)) ; WHAT IN THE + (first arguments)) + (t + (apply (symbol-function head) + (mapcar #'eval arguments)))))) + ; ... + (t (error "What the hell is this?")))) +``` + +That `(quote quote)` looks just completely bonkers, right? Step through it +piece by piece to figure it out: + +* What do the five characters `quote` read as? What do they evaluate to? +* What do the thirteen (spooky!) characters `(quote quote)` read as? When + `eval` gets its hands on that, what clause in the `cond` does it hit? What + does that return? + +Try these in a Lisp REPL if you're not 100% sure of the answers. The functions +`read-from-string`, `eval`, and `type-of` will come in handy. + +### The Read Macro + +Now let's look at the final piece of the puzzle. `'` (that's a single ASCII +quote character) is a [read macro][]. Don't worry if you don't know exactly +what that means right now. The important part is that when `read` sees a quote +character, it returns a two-element list containing the symbol `quote` and the +next form, which is read normally. + +The four characters `'foo` go into `read` and come out as a two-element list +containing: the symbol `quote` and the symbol `foo`. Again, because it's +important: this happens *at read time*. By the time `eval` sees anything it's +already been turned into that two-element list. + +It can be a bit slippery to get a grip on this, because `print` will "helpfully" +print a list of `(quote whatever)` as `'whatever`! But we can use trusty old +`car` and `cdr` to see what's really going on: + +```lisp +CL-USER> (read-from-string "'foo") + +'FOO + +CL-USER> (type-of (read-from-string "'foo")) + +CONS + +CL-USER> (car (read-from-string "'foo")) + +QUOTE + +CL-USER> (cdr (read-from-string "'foo")) + +(FOO) +``` + +That's really all there is to it: `'form` is returned by `read` as `(quote +form)`. Then that gets passed along to `eval`, which sees the `quote` special +operator and passes the argument back untouched. + +Note that `form` doesn't have to be a symbol: + +* `'(1 2 (4 5))` reads as `(quote (1 2 (4 5)))` +* `'150` reads as `(quote 150)` +* `'"hello"` reads as `(quote "hello")` +* `'(if if if if)` reads as `(quote (if if if if))` + +[read macro]: https://gist.github.com/chaitanyagupta/9324402 + +### Exercises + +That pretty much wraps up `quote`. Once you've cleanly separated `read`, +`eval`, and `print` in your mind it becomes a lot less mystical (or perhaps +*more*, depending on your point of view). + +If you want to twist your brain just a bit more, try these exercises: + +1. What do the characters `foo` read in as? What do they evaluate to? How does it print? +2. What do the characters `'foo` read in as? What do they evaluate to? How does it print? +3. What do the characters `''foo` read in as? What do they evaluate to? How does it print? +4. What do the characters `'1.0` read in as? What do they evaluate to? How does it print? +5. What do the characters `'(1 2 3)` read in as? How many elements are in the list? What does it evaluate to? How many elements are in *that* list? +6. Update our implementation of `eval` to use the `'` read macro instead of `quote`. +7. Certain symbols like `t` and `nil` are special — they evaluate to themselves like numbers or strings. Add this to `eval`. +8. What do the characters `nil` read in as? What is the type of that object? What do they evaluate to? What type is that result? +9. What do the characters `'nil` read in as? What is the type of that object? What do they evaluate to? What type is that result? +10. What do the characters `''nil` read in as? What is the type of that object? What do they evaluate to? What type is that result? + +## A Quick Recap + +If you've gotten this far and understood everything, you should have a good +grasp of how symbolic computation works. Here are the main things I hope you've +taken away from this post: + +* `read`, `eval`, and `print` are three separate, distinct functions/phases in + the interpretation of Lisp (and really *all* (okay, *most*)) code. +* Keeping these three phases *clearly separated* in your mind will make it + easier to understand symbols and quoting in Lisp. +* Symbols are nothing magic, they're just a particular data structure with + a couple of fields. +* The important thing is how `read` and `eval` *treat* symbols. That's where + the magic happens. +* `quote` is just a "short-circuit" in `eval` that turns out to be really + useful. +* `'` is just a lazier way of writing `(quote ...)`. + +## Where to Go From Here + +Now that you've got a firmer grasp on what the hell symbols actually *are*, +there are a lot of cool things you might want to check out that will show you +what you can do with them: + +* Reread some of your introductory Lisp books again, seeing if things seem to + make a bit more sense now. +* Read about backquote/quasiquote. How does it compare to the `quote` we + explored here? +* Read [Paradigms of Artificial Intelligence Programming][PAIP] and pay + attention to how Norvig uses symbols for various tasks. Also pay attention to + what he says about property lists, because we skipped them here. +* Read the [guide to the Common Lisp package system][packaging] to learn all the + stuff I left out about the Common Lisp package system. +* Gaze upon [this macro][symbolicate] in wonder (or horror). +* Read the HyperSpec pages for things like `symbol` and `quote`. +* Find and read your Lisp implementation's `eval` function. +* Find and read *another* Lisp implementation's `eval` function. +* Watch the [SICP Lectures][SICP] on YouTube. + +[PAIP]: http://www.amazon.com/dp/1558601910/?tag=stelos-20 +[symbolicate]: https://github.com/zkat/squirl/blob/f0dc57dfee728df94b2a35956e18e143e8b8d275/src/vec.lisp#L28-L38 +[packaging]: https://www-fourier.ujf-grenoble.fr/~sergerar/Papers/Packaging.pdf +[SICP]: http://www.amazon.com/dp/0262510871/?tag=stelos-20 + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2016/08/lisp-jam-postmortem.html --- a/content/blog/2016/08/lisp-jam-postmortem.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,981 +0,0 @@ - {% extends "_post.html" %} - - {% load mathjax %} - - {% hyde - title: "August 2016 Lisp Game Jam Postmortem" - snip: "Porting a game from Clojure to Common Lisp." - created: 2016-08-15 13:45:00 - %} - - {% block article %} - -The [August 2016 Lisp Game Jam][] just wrapped up at the end of last week. -I had some free time so I decided to take part, but I did something a bit -different. Instead of making a new game I ported an existing one ([Silt][]) to -Common Lisp. - -I once read somewhere that when trying to build things and learn programming -languages you should either build something you know in a language you're -learning, or build something new in a language you already know, but *not* try -to do both at the same time. I've been getting into Common Lisp over the past -year, so for this game jam I decided to port my [Ludum Dare 34 game][] from -Clojure to Common Lisp. - -The game jam was ten days long. I didn't work on the game every day, but I did -manage to finish porting it over. I improved and polished a few mechanics along -the way, learned a lot, and ended up with a nice little library that sprung out -of the code. I'm happy with the result. - -The code is [on Bitbucket][Silt 2]. You can play the game over telnet if you -want to try it out: `telnet silt.stevelosh.com`. In this post I'm just going to -jot down a few things I found interesting. - -Disclaimer: I'm going to simplify some of the code snippets to make them easier -to read. If you want the full details you can read the actual code. - -[August 2016 Lisp Game Jam]: https://itch.io/jam/august-2016-lisp-game-jam -[Ludum Dare 34 game]: /blog/2015/12/ludum-dare-34/ -[Silt]: http://bitbucket.org/sjl/silt/ -[Silt 2]: http://bitbucket.org/sjl/silt2/ - -[TOC] - -## Development - -[Silt 2][] is written in Common Lisp. It uses [cl-charms][] (a wrapper around -[ncurses][]) to handle drawing to the terminal, and a few other Common Lisp -libraries like [iterate][] and [cl-arrows][]. - -I developed it on [SBCL][] and OS X, and the telnet server is running Debian so -it works there too. It almost runs in [ClozureCL][], but something -Unicode-related is broken with ncurses under CCL and I didn't bother debugging -it. - -I used [Roswell][] to build a standalone binary for "releases". This binary -starts up much faster than loading everything from scratch. - -I use [Neovim][] and was pleasantly surprised when running ncurses inside -Neovim's terminal emulator Just Worked (especially since the cl-charms `README` -specifically says you *can't* run it in emacs' terminal!). It was really nice to -have the actual game running inside my text editor. - -[iterate]: https://common-lisp.net/project/iterate/ -[cl-arrows]: https://github.com/nightfly19/cl-arrows -[cl-charms]: https://github.com/HiTECNOLOGYs/cl-charms -[ncurses]: https://en.wikipedia.org/wiki/Ncurses -[SBCL]: http://www.sbcl.org/ -[ClozureCL]: http://ccl.clozure.com/ -[Roswell]: https://github.com/roswell/roswell -[Neovim]: https://neovim.io/ - -## ncurses and cl-charms - -[cl-charms][] is a wrapper around [ncurses][] that I used to handle drawing the -game to the terminal. The original Clojure version used [clojure-lanterna][]. - -The game's drawing code is pretty simple, so there's not a whole lot to say -here. I loop over the screen, drawing the contents of each world coordinate at -each screen coordinate, and refresh the window. - -cl-charms mostly worked out great. It's a bit wordy at times (always having to -pass `charms:*standard-window*` to everything), but you can wrap it up pretty -easily. I'd recommend it if you need to do console drawing in Common Lisp. - -cl-charms has a low-level interface that's just an FFI wrapper around ncurses, -and a high-level interface that abstracts some of the Cishness away for you. -I mostly used the high-level interface, but one big thing that's missing is -support for colors. Working with colors in ncurses is a bit tedious, but this -is Lisp so I can just abstract away all the boring stuff: - - :::text - (defmacro defcolors (&rest colors) - `(progn - ,@(iterate (for n :from 0) - (for (constant nil nil) :in colors) - (collect `(define-constant ,constant ,n))) - (defun init-colors () - ,@(iterate - (for (constant fg bg) :in colors) - (collect `(charms/ll:init-pair ,constant ,fg ,bg)))))) - - (defcolors - (+color-white-black+ charms/ll:COLOR_WHITE charms/ll:COLOR_BLACK) - (+color-blue-black+ charms/ll:COLOR_BLUE charms/ll:COLOR_BLACK) - (+color-cyan-black+ charms/ll:COLOR_CYAN charms/ll:COLOR_BLACK) - (+color-yellow-black+ charms/ll:COLOR_YELLOW charms/ll:COLOR_BLACK) - (+color-green-black+ charms/ll:COLOR_GREEN charms/ll:COLOR_BLACK) - (+color-pink-black+ charms/ll:COLOR_MAGENTA charms/ll:COLOR_BLACK) - - (+color-black-white+ charms/ll:COLOR_BLACK charms/ll:COLOR_WHITE) - (+color-black-yellow+ charms/ll:COLOR_BLACK charms/ll:COLOR_YELLOW) - - (+color-white-blue+ charms/ll:COLOR_WHITE charms/ll:COLOR_BLUE) - - (+color-white-red+ charms/ll:COLOR_WHITE charms/ll:COLOR_RED) - - (+color-white-green+ charms/ll:COLOR_WHITE charms/ll:COLOR_GREEN)) - - (defmacro with-color (color &body body) - (once-only (color) - `(unwind-protect - (progn - (charms/ll:attron (charms/ll:color-pair ,color)) - ,@body) - (charms/ll:attroff (charms/ll:color-pair ,color))))) - - -[clojure-lanterna]: http://sjl.bitbucket.org/clojure-lanterna/ - -## Using a State Machine as the Game Loop - -One thing many games have in common is a [game loop][]. The original version of -Silt had one, but for the rewrite I decided to structure the main flow of the -game as a state machine instead. This worked out really well and I'm glad I did -it. - -At first I looked around and tried to find a state machine library for Common -Lisp, but then I realized I was being ridiculous and could just model a state -machine with vanilla Lisp functions: - - :::text - (defun state-title () - (render-title) - (press-any-key) - (state-intro)) - - (defun state-intro () - (render-intro) - (press-any-key) - (state-generate)) - - (defun state-generate () - (render-generate) - (reset-world) - (generate-world) - (state-map)) - - (defun state-map () - (charms:enable-non-blocking-mode charms:*standard-window*) - (state-map-loop)) - - (defun state-map-loop () - (case (handle-input-map) - ((:quit) (state-quit)) - ((:regen) (state-generate)) - ((:help) (state-help)) - (t (progn - (unless *paused* - (iterate (repeat *frame-skip*) - (tick-world) - (tick-log))) - (render-map) - (when *sleep* - (sleep 0.05)) - (state-map-loop))))) - - (defun state-help () - (render-help) - (press-any-key) - (state-map)) - - (defun state-quit () - 'goodbye) - -This worked especially well with cl-charms and ncurses because for states like -the title and help screens there's no point in looping to redraw the screen over -and over again while waiting for input. I just flipped ncurses into -block-while-awaiting-input mode and let it free up the CPU while waiting for the -user to continue. - -In hindsight I probably should have split out the pause state into a separate -state, which would have let me use blocking input there too. - -Using functions for states like this is only possible because SBCL (and CCL) -perform [last call optimization][], so the stack doesn't get blown by all the -recursion happening. - -[game loop]: https://en.wikipedia.org/wiki/Game_programming#Game_structure -[last call optimization]: https://en.wikipedia.org/wiki/Tail_call - -## Terrain Generation - -The original Silt was made for Ludum Dare 34 in 72 hours, so I didn't spend too -much time on terrain. I just created an empty world and scattered some lakes -around it, which looked like this: - -[![Screenshot of terrain in the original game](/media/images{{ parent_url }}/silt1-terrain.png)](/media/images{{ parent_url }}/silt1-terrain.png) - -This worked and was quick, but is pretty boring and ugly. In the past few -months I've learned a lot more about terrain generation, so I fleshed things out -a bit more for the new port: - -[![Screenshot of terrain in the new version](/media/images{{ parent_url }}/silt2-terrain.png)](/media/images{{ parent_url }}/silt2-terrain.png) - -Now I've got oceans and mountains for the creatures to explore. - -### Tiling Diamond Square - -My initial impulse was to use [Perlin Noise][] or [Simplex Noise][] to generate -the heightmap for the world, but I ran into a problem. I wanted the world to be -a torus, just like in the original game, so I needed a terrain generation -algorithm that would generate tileable/wrappable heightmaps. - -One way to do this is to use higher-dimensional noise to get 2D noise that -tiles. If you want to get a 2D heightmap that's tileable in one direction, you -can use 3D noise and take a cylindrical slice of it. To get a heightmap that -tiles both ways you need to use 4D noise. [This article][ron-noise] gives -a really nice overview of the process. - -Unfortunately I couldn't find an implementation of 4D Simplex Noise in Common -Lisp. [black-tie][] and [noise][] both only offer up to 3D noise, and I don't -feel confident enough to implement it myself, even after skimming the simplex -noise paper. - -So I decided to try a different approach and figure out how to modify [Diamond -Square][] to tile. The [Wikipedia article for Diamond Square][ds-wiki] says: - -> Another option [for the diamond step] is to 'wrap around', taking the fourth -> value from the other side of the array. When used with consistent initial -> corner values this method also allows generated fractals to be stitched -> together without discontinuities. - -This sounded great, but after thinking about it for a bit it's obviously not -correct. If we have a heightmap and do what the article says, it will seem to -work at first: - -
-                      ╔══════════════════╗
-    ┌─┬─┬─┬─┬─┐       ║   ┌─┬─┬─┬─┬─┐    ║
-    │5│ │ │ │5│       ║   │5│ │ │ │5│    ║
-    ├─┼─┼─┼─╱┬╲       ║   ├─┼─┼─┼─╱┬╲    ║
-    │ │ │ │╱│││╲      ║   │ │ │ │╱│││╲   ║
-    ├─┼─┼─╱─┼▼┤ ╲     ║   ├─┼─┼─╱─┼▼┤ ╲  ║
-    │ │ │3├─▶◉◀──?    ╚═══════│3├─▶◉◀════╝
-    ├─┼─┼─╲─┼▲┤ ╱         ├─┼─┼─╲─┼▲┤ ╱
-    │ │ │ │╲│││╱          │ │ │ │╲│││╱
-    ├─┼─┼─┼─╲┴╱           ├─┼─┼─┼─╲┴╱
-    │5│ │ │ │5│           │5│ │ │ │5│
-    └─┴─┴─┴─┴─┘           └─┴─┴─┴─┴─┘
-
- -Wrapping like this will indeed make sure that the averages match up, but there's -two problems. - -First: the corners are all the same value, which means that when you put four -heightmaps next to each other there's an unnatural flat area of four identical -height values next to each other. This probably wouldn't be noticeable in -practice, but if you want to do things *right* it won't be acceptable. - -But the *real* problem is the jitter. If the jitter on one side of the map -happens to be large and positive and the jitter on the other side happens to be -large and negative, you'll get a jarring "cliff" when you try to tile them: - -[![Example of poorly-tiling diamond square](/media/images{{ parent_url }}/bad-tiling-ds.png)](/media/images{{ parent_url }}/bad-tiling-ds.png) - -The solution I came up with is to reduce the size of the heightmap by 1. -Instead of the heightmap being \\(2^n + 1\\) in each dimension we can make it -\\(2^n\\) and adjust the coordinate-wrapping function appropriately. -Importantly, we *don't* change the calculation of the radius values as we -iterate over the array, so this means quite often we'll be "reaching" for that -final row/column: - -
-     ?       ?
-    ┌─╲─┬─┬─╱
-    │ │╲│ │╱│
-    ├─┼─◢─◣─┤
-    │ │ │◉│ │
-    ├─┼─◥─◤─┤
-    │ │╱│ │╲│
-    ├─╱─┼─┼─╲
-    │5│ │ │ │?
-    └─┴─┴─┴─┘
-
- -When we try to access that nonexistent coordinate we just wrap around back to -zero. Notice that we also only need to initialize a single corner cell now. - -It's a simple change, but the result is *much* nicer: - -[![Example of nicely-tiling diamond square](/media/images{{ parent_url }}/good-tiling-ds.png)](/media/images{{ parent_url }}/good-tiling-ds.png) - -[Perlin Noise]: https://en.wikipedia.org/wiki/Perlin_noise -[Simplex Noise]: https://en.wikipedia.org/wiki/Simplex_noise -[ron-noise]: http://ronvalstar.nl/creating-tileable-noise-maps -[black-tie]: https://github.com/aerique/black-tie -[noise]: https://github.com/sebity/noise -[Diamond Square]: /blog/2016/06/diamond-square/ -[ds-wiki]: https://en.wikipedia.org/wiki/Diamond-square_algorithm - -## Entity, Aspects, and Systems - -Terrain generation is pretty, but the next step in the port was to add some -plants, creatures, and artifacts. In the original game I just represented -things in the world as vanilla Clojure maps, but that was getting kind of messy -and I wanted to try a different approach this time. - -Recently I read through [Game Engine Architecture][] (a *fantastic* book) and -made a few games in [Unity][], which together made me want to try using an -[Entity/Component System][] this time around. There are a couple of ECS -libraries out there for Common Lisp like [cl-ecs][] and [ecstasy][], but in true -Lisp fashion I ended up not being quite satisfied with any of them and writing -Yet Another God Damn Library. - -It's called [Beast][]. It's subtly different than the others in that it prefers -to be a really thin layer over CLOS and uses inheritance instead of composition. -It uses the word "aspect" instead of "component" to try to overload that word -a bit less, so it's the "Basic Entity/Aspect/System Toolkit". It ended up being -about 150 lines of code (not including docstrings), so I managed to avoid going -down too much of a rabbit hole during the jam. - -If you want to know all the details, check out its documentation (it has -*actual* documentation). But here I'll just talk about a couple of the -particular bits of Silt that I used it for. - -[Unity]: https://unity3d.com/ -[Game Engine Architecture]: http://www.amazon.com/dp/1466560010/?tag=stelos-20 -[Entity/Component System]: https://en.wikipedia.org/wiki/Entity_component_system -[cl-ecs]: https://github.com/lispgames/cl-ecs -[ecstasy]: https://github.com/mfiano/ecstasy -[beast]: http://sjl.bitbucket.org/beast/overview/ - -### Coordinates - -The first thing I needed was a way to keep track of where things are in the -world. - -If the world space were continuous a [quadtree][] would have been my first -choice, but in Silt the world is split into discrete integer coordinates. -Creatures move directly from \\((x, y)\\) to \\((x+1, y+1)\\). I decided to use -a simple array of lists to represent this: - - :::text - (defparameter *coords-contents* - (make-array (list +world-size+ +world-size+) - :initial-element nil)) - -Each value in the array is a list of the entities that are currently there. -This means looking up what things are at a given coordinate is a single fast -`aref`. - -I tried using a hash table instead of an array at first, thinking that if the -world were fairly sparse it would be wasteful to allocate an array with a ton -of `nil` values in it. But the array method is much faster for looking things -up (which happens a lot) and memory is cheap, so I decided against the hash -tables. It worked great in the end. - -Entities need to know where they are in the world, so I defined a Beast aspect -for that: - - :::text - (define-aspect coords x y) - -Then I defined a few functions to handle moving entities into, out of, and -around the world: - - :::text - (defun coords-insert-entity (e) - (push e (aref *coords-contents* (coords/x e) (coords/y e)))) - - (defun coords-remove-entity (e) - (zap% (aref *coords-contents* (coords/x e) (coords/y e)) - #'delete e %)) - - (defun coords-move-entity (e new-x new-y) - (coords-remove-entity e) - (setf (coords/x e) (wrap new-x) - (coords/y e) (wrap new-y)) - (coords-insert-entity e)) - - (defun coords-lookup (x y) - (aref *coords-contents* (wrap x) (wrap y))) - -Entities might also like to know what's near them: - - :::text - (defun nearby (entity &optional (radius 1)) - (remove entity - (iterate - outer - (with x = (coords/x entity)) - (with y = (coords/y entity)) - (for dx :from (- radius) :to radius) - (iterate - (for dy :from (- radius) :to radius) - (in outer - (appending (coords-lookup (+ x dx) - (+ y dy)))))))) - -This ends up compiling down to a nice tight loop of \\((2 * radius + 1)^2\\) -`aref`s. I only wish iterate had a nicer syntax for looping over nested -indices like this. I'm sure it's possible to write an iterate driver for it -- -maybe someday I'll try making one. - -I also needed a way to get entities into the world array when they're created -and remove them when they die. Beast (well, actually CLOS) makes this trivially -easy with auxiliary methods: - - :::text - (defmethod entity-created :after ((entity coords)) - (coords-insert-entity entity)) - - (defmethod entity-destroyed :after ((entity coords)) - (coords-remove-entity entity)) - -[quadtree]: https://en.wikipedia.org/wiki/Quadtree - -### User Interface - -Once I had a way of know where things are, the next step was to display them on -the screen. I broke this into a few separate aspects. - -#### Visible - -The `visible` aspect is for things that are drawn on the screen with -a particular glyph and color: - - :::text - (define-aspect visible glyph color) - - ;; ... - - (define-entity tree (coords visible ...)) - - (defun make-tree (x y) - (create-entity 'tree - :coords/x x - :coords/y y - :visible/glyph "T" - :visible/color +color-green-black+ - ;; ... - )) - - -The drawing code can then figure out what to draw for each screen coordinate: - - :::text - (defun draw-map () - (iterate - (for sx :from 0 :below *screen-width*) - (for wx :from *view-x*) - (iterate - (for sy :from 0 :below *screen-height*) - (for wy :from *view-y*) - (for entity = (find-if #'visible? (coords-lookup wx wy))) - (if entity - (with-color (visible/color entity) - (write-string-at (visible/glyph entity) sx sy)) - ;;; otherwise draw the terrain - (...))))) - -Again: my kingdom for a `(for-nested ...)` iterate driver! But the core is just -using `(find-if #'visible? (coords-lookup wx wy))` to find the first visible -thing and then drawing it: - -[![Screenshot of entities with the visible aspect](/media/images{{ parent_url }}/aspect-visible.png)](/media/images{{ parent_url }}/aspect-visible.png) - -I used `find-if` instead of `remove-if-not` because we can only draw one -character to a given position in the terminal anyway, so I just pick the first -thing that happens to be in the list. - -#### Flavor - -The `flavor` aspect is for adding [flavor text][] that appears when the user -puts their cursor over an entity: - - :::text - (define-aspect flavor text) - - ;; ... - - (define-entity tree (coords visible flavor ...)) - - (defun make-tree (x y) - (create-entity 'tree - :coords/x x - :coords/y y - :visible/glyph "T" - :visible/color +color-green-black+ - :flavor/text - '("A tree sways gently in the wind."))) - -Then when the user's cursor is at a certain position I can find all the entities -there and draw the flavor text for any that have the `flavor` aspect: - - :::text - (defun draw-selected () - (write-left - (iterate - (for entity :in (multiple-value-call #'coords-lookup - (screen-to-world *cursor-x* *cursor-y*))) - (when (typep entity 'flavor) - (appending (flavor/text entity) :into text) - - ;; ... - - (collecting "" :into text)) - (finally (return text))) - 1 1 :pad t)) - -Which looks like this: - -[![Screenshot of flavor text](/media/images{{ parent_url }}/aspect-flavor.png)](/media/images{{ parent_url }}/aspect-flavor.png) - -Of course the flavor text doesn't have to be a constant: - - :::text - (defun make-creature (x y &key - (color +color-white-black+) - (glyph "@")) - (let ((name (random-name))) - (create-entity 'creature - :name name - :coords/x x - :coords/y y - :visible/color color - :visible/glyph glyph - :flavor/text - (list (format nil "A creature named ~A is here." name) - "It likes food.")))) - -[flavor text]: https://en.wikipedia.org/wiki/Flavor_text - -#### Inspectable - -The last thing I wanted was an easy way to show attributes of entities in the -main game UI. The original Clojure game just dumped the entire object to the -screen: - -[![Screenshot of creature inspection in the original game](/media/images{{ parent_url }}/silt1-inspect.png)](/media/images{{ parent_url }}/silt1-inspect.png) - -But this time I wanted a bit more control. The `inspectable` aspect has a list -of things that should be displayed. These can be symbols (which denote CLOS -slot names) or functions that return `(label . text)` conses: - - :::text - (define-aspect inspectable - (slots :initform nil)) - - (defun inspectable-get (entity slot) - (etypecase slot - (symbol (cons slot (slot-value entity slot))) - (function (funcall slot entity)))) - -When creating an entity I can just list out the slots I want to be displayed on -the screen, or use a little `lambda` if I want to show something that's not an -actual slot: - - :::text - (defun make-fruit (x y) - (create-entity 'fruit - ;; ... - :inspectable/slots '(edible/energy))) - - (defun make-creature (x y &key ...) - (let ((name (random-name))) - (create-entity 'creature - ;; ... - :inspectable/slots - (list 'name - (lambda (c) (cons 'directions ...)) - 'metabolizing/energy - 'metabolizing/insulation - 'aging/birthtick - 'aging/age)))) - -Then I just append some extra text for `inspectable` entities when drawing -descriptions of things at the cursor position: - - :::text - (defun draw-selected () - (write-left - (iterate - (for entity :in (multiple-value-call #'coords-lookup - (screen-to-world *cursor-x* *cursor-y*))) - (when (typep entity 'flavor) - ;; ... - (when (typep entity 'inspectable) - (appending - (indent - (iterate - (with slots = (mapcar (curry #'inspectable-get entity) - (inspectable/slots entity))) - (with width = (apply #'max - (mapcar (compose #'length #'symbol-name #'car) - slots))) - (for (label . contents) :in slots) - (collect - (let ((*print-pretty* nil)) - (format nil "~vA ~A" width label contents))))) - :into text)) - - (collecting "" :into text)) - (finally (return text))) - 1 1 :pad t)) - -This is pretty ugly because I wanted to justify and indent things nicely, but -the result looks much nicer than the original game: - -[![Screenshot of creature inspection in the new version](/media/images{{ parent_url }}/silt2-inspect.png)](/media/images{{ parent_url }}/silt2-inspect.png) - -### Food - -Seeing the world is nice, but we also want the things in it to actually *do* -something. The world revolves heavily around food and energy, so I defined -a few aspects to handle things: - - :::text - (define-aspect edible - energy - original-energy) - - (define-aspect decomposing - rate - (remaining :initform 1.0)) - - (define-aspect fruiting - chance) - - (defmethod initialize-instance :after ((e edible) &key) - (setf (edible/original-energy e) - (edible/energy e))) - -I do wish there was a slightly less wordy way to default the value of one slot -to another one, but oh well. - -Then I just added the aspects to the appropriate entities: - - :::text - (define-entity tree (coords visible fruiting flavor)) - (define-entity fruit (coords visible edible flavor decomposing inspectable)) - (define-entity algae (coords visible edible decomposing)) - (define-entity grass (coords visible edible decomposing)) - (define-entity corpse (coords visible flavor decomposing)) - -Trees can grow fruit, so they have the `fruiting` aspect. The `grow-fruit` -Beast system handles growing some each tick: - - :::text - (define-system grow-fruit ((entity fruiting coords)) - (when (randomp (fruiting/chance entity)) - (make-fruit (wrap (random-around (coords/x entity) 2)) - (wrap (random-around (coords/y entity) 2))))) - -Fruit is `edible`, but also decomposes over time. It's got `flavor` and -`inspectable` aspects so you can see how much energy is left. - -I added algae and grass as secondary food sources to spread out the food supply -a bit more and make the creatures a bit less dependent on the trees. I didn't -give these flavor text to avoid cluttering up the UI too much. - -I considered making corpses edible too, but figured that might be a bit too -gruesome. So corpses decompose, but the critters aren't cannibals. - -I made a couple of Beast systems to handle the process of decomposing things -every game tick: - - :::text - (define-system rot ((entity decomposing)) - (when (minusp (decf (decomposing/remaining entity) - (decomposing/rate entity))) - (destroy-entity entity))) - - (define-system rot-food ((entity decomposing edible)) - (setf (edible/energy entity) - (lerp 0.0 (edible/original-energy entity) - (decomposing/remaining entity)))) - -`rot` runs on everything with the `decomposing` aspect. It ticks along the -progress of an entity's decomposition, and destroys it once it's finished. - -`rot-food` runs on every entity that's both `decomposing` *and* `edible`. It -reduces the energy value of the food over time, because rotten food is less -healthy. I'm pretty happy with how easy Beast makes this kind of thing. - -### Creatures and Mysteries - -The final pieces of the world are the creatures and artifacts. - -#### Energy -Creatures need food (energy) to survive. I modeled this with a `metabolizing` -aspect and `consume-energy` system: - - :::text - (define-aspect metabolizing insulation energy) - - (defmethod starve ((entity entity)) - (destroy-entity entity)) - - (defmethod calculate-energy-cost ((entity metabolizing)) - (let* ((insulation (metabolizing/insulation entity)) - (base-cost 1.0) - (temperature-cost (max 0 (* 0.2 (- (abs *temperature*) insulation)))) - (insulation-cost (* 0.1 insulation))) - (+ base-cost temperature-cost insulation-cost))) - - (define-system consume-energy ((entity metabolizing)) - (when (minusp (decf (metabolizing/energy entity) - (calculate-energy-cost entity))) - (starve entity))) - -I made `starve` and `calculate-energy-cost` generic functions because I thought -I might eventually have different metabolizing things in the world and might -want to override them. I didn't end up doing this in the end (creatures are the -only things that burn energy) so these could have been normal functions. - -The energy mechanic works similarly to the original game: - -* Creatures spend a bit of energy each tick to stay alive. -* When you make the temperature hotter or colder, it costs additional energy per - tick for the creatures to live. -* Creatures sometimes gain/lose insulation during reproduction, which mitigates - the energy cost of the temperature difference. -* Insulation itself costs a little bit of energy every tick. - -The effect is that if you change the temperature gradually over time, the -population will evolve higher insulation values (because the children with more -insulation are more likely to survive longer). If you then set the temperature -back to zero (the ideal) the population will eventually evolve to shed the -insulation, because it costs a little bit of energy and doesn't provide any -benefit when the world is pleasant. Natural selection is fun. - -The last piece of the puzzle is letting things actually take action. Creatures -and some artifacts need to take an action on every tick, while other artifacts -only do things occasionally. A pair of aspects and systems handles the -bookkeeping here: - - :::text - (define-aspect sentient function) - (define-aspect periodic - function - (counter :initform 1) - next - min - max) - - (define-system sentient-act ((entity sentient)) - (funcall (sentient/function entity) entity)) - - (define-system periodic-tick ((entity periodic)) - (when (zerop (setf (periodic/counter entity) - (mod (1+ (periodic/counter entity)) - (periodic/next entity)))) - (setf (periodic/next entity) - (random-range (periodic/min entity) - (periodic/max entity))) - (funcall (periodic/function entity) entity))) - -I'm not going to go over all the actual AI and actions, you can take a look at -the code if you're curious. - -## Random Name Generation - -I wanted to add a more personal connection to the creatures this time around, so -I decided they should have names. I used a really simple form of -[syllable-based name generation][namegen] to give each creature its own random -name: - - :::text - (defparameter *name-syllables* - (-> "syllables.txt" - slurp - read-from-string - (coerce 'vector))) - - (defun random-name () - (format nil "~:(~{~A~}~)" - (iterate (repeat (random-range 1 5)) - (collect (random-elt *name-syllables*))))) - -To get a random name I just smash together one to four random syllables. To -make a list of syllables I grabbed some Icelandic text and made a pair of really -janky shell and Python scripts to print out every 3/4/5-letter chunk of every -word, sort them by frequency, and take the top 500. - -They're not *really* syllables but they're okay for just a couple of lines of -code and a few minutes work: - -[![Screenshot of creature names](/media/images{{ parent_url }}/silt-names.png)](/media/images{{ parent_url }}/silt-names.png) - -[namegen]: http://www.roguebasin.com/index.php?title=Syllable-based_name_generation - -## Simple Data Structures - -As I coded things up I wound up with a handy pair of data structures I might use -again for other things in the future. - -### Weightlists - -Each game tick a creature needs to decide which direction to walk. At the start -of the game they just pick a random direction, but as they reproduce their -children can mutate to prefer certain directions over others. - -The natural selection of Silt turns out to prefer creatures that wander around -to those that stay in place. Fruit takes time to grow, so it's more effective -to travel around and gather it than to sit in place waiting for it to regrow. - -The original game just used a Clojure vector of weights and directions to -represent how much a creature prefers each direction. That worked, but the -weights are only ever set/changed when a creature is born, and a random element -is chosen every turn. It's more efficient in the long run if we precompute -a few things up front, so I made a little "weightlist" API: - - :::text - (defstruct (weightlist (:constructor %make-weightlist)) - weights sums items total) - - (defun make-weightlist (items weights) - "Make a weightlist of the given items and weights." - (%make-weightlist - :items items - :weights weights - :sums (prefix-sums weights) - :total (apply #'+ weights))) - - (defun weightlist-random (weightlist) - "Return a random item from the weightlist, taking the weights into account." - (iterate - (with n = (random (weightlist-total weightlist))) - (for item :in (weightlist-items weightlist)) - (for weight :in (weightlist-sums weightlist)) - (finding item :such-that (< n weight)))) - -This is pretty straightforward. Note that the weights can be integers or floats -(or some of each!) and things will Just Work, because Common Lisp's `random` can -take either. Weights of zero are fine too, as long as at least one element has -a nonzero weight. - -### Ticklists - -In a couple of places I needed some kind of list where items in it expire over -time. For example: - -* The Fountain artifact only lets creatures drink from it once every thousand - ticks, so I needed a way to keep track of the entities that had drank - recently. -* The game log at the bottom of the screen contains messages that should be - shown for a certain number of ticks, then disappear. - -I made a simple little thing I called a "ticklist" to handle these: - - :::text - (defun make-ticklist () - nil) - - (defmacro ticklist-push (ticklist value lifespan) - `(push (cons ,lifespan ,value) ,ticklist)) - - (defun ticklist-tick (ticklist) - (flet ((decrement (entry) - (decf (car entry))) - (dead (entry) - (minusp (car entry)))) - (->> ticklist - (mapc #'decrement) - (remove-if #'dead)))) - - (defun ticklist-contents (ticklist) - (mapcar #'cdr ticklist)) - -Internally a ticklist is just a list of `(remaining-ticks . thing)` conses, but -the rest of my code doesn't have to care about that: - - :::text - (defun fountain-act (f) - (with-slots (recent) f - (zapf recent #'ticklist-tick) - (iterate - (with already-drank = (ticklist-contents recent)) - (for creature :in (remove-if-not #'creature? (nearby f))) - (unless (member creature already-drank) - (creature-mutate-appearance creature) - (ticklist-push recent creature 1000) - (log-message "~A drinks from the fountain and... changes." - (creature-name creature)))))) - - (defun log-message (s &rest args) - (ticklist-push *game-log* (apply #'format nil s args) 200)) - - (defun state-map-loop () - ;; ... - (unless *paused* - (iterate (repeat *frame-skip*) - (tick-world) - (zapf *game-log* #'ticklist-tick)))) - -## Profiling and Performance - -When something starts slowing things down it's helpful to be able to turn on -profiling and see what's going on. SBCL has a nice statistical profiler, so -I made a couple of functions to flip it on and off as needed: - - :::text - #+sbcl - (defun dump-profile () - (with-open-file (*standard-output* "silt.prof" - :direction :output - :if-exists :supersede) - (sb-sprof:report :type :graph - :sort-by :cumulative-samples - :sort-order :ascending) - (sb-sprof:report :type :flat - :min-percent 0.5))) - - #+sbcl - (defun start-profiling () - (sb-sprof::reset) - (sb-sprof::profile-call-counts "SILT") - (sb-sprof::start-profiling :max-samples 50000 - ; :mode :cpu - :mode :time - :sample-interval 0.01 - :threads :all)) - - #+sbcl - (defun stop-profiling () - (sb-sprof::stop-profiling) - (dump-profile)) - -When I wanted to check performance I could just evaluate `(start-profiling)` -over NREPL and let the game continue to run, then `(stop-profiling)` a little -bit later and look at the results. It came in handy once or twice when tracking -down some slowness. - -## Future Improvements and Ideas - -This game jam was quite a bit of fun! I'm happy with the results and feel like -I've learned a lot along the way. - -I'm done with the game and don't plan on updating it any more, but I'll scribble -down a few extra ideas for things that could be improved here, just to get them -out of my head: - -* Figure out the Unicode issues with cl-charms and CCL. -* Contribute to cl-charms to add some higher-level tools for working with color. -* Contribute to one of the Common Lisp noise libraries to implement the 4D - variant of Simplex Noise. -* Flesh out the name generation into something much nicer and more polished. -* Implement different costs for moving over different terrain. -* Add health, fighting, and carnivores. -* Add more mysterious artifacts to the world. -* Flesh out the vegetation model to let trees grow and die, algae spread, etc. -* Improve the visuals. [Brogue][] proves you can do far more than you might - think with just Unicode characters. -* Model senses like vision, providing the creatures with more information but - with an energy cost. -* Give creatures "brains" by generating and mutating actual Lisp code. This - would let the creatures learn strategies over time, though I'm not sure how - feasible it would be. -* Improve performance by profiling much more and fixing the hottest parts of the - code. -* Add saving and loading of the world. -* Add the ability to seed the RNG and make everything deterministic, so people - can share interesting seeds. Doing this for the terrain generation at least - should be pretty easy, the game as a whole might be slightly trickier. -* Improve the UI a bit more, maybe using ncurses' support for windows layered on - top of other windows. - -[Brogue]: https://sites.google.com/site/broguegame/ - - {% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2016/08/lisp-jam-postmortem.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2016/08/lisp-jam-postmortem.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,1003 @@ ++++ +title = "August 2016 Lisp Game Jam Postmortem" +snip = "Porting a game from Clojure to Common Lisp." +date = 2016-08-15T13:45:00Z +draft = false + ++++ + +The [August 2016 Lisp Game Jam][] just wrapped up at the end of last week. +I had some free time so I decided to take part, but I did something a bit +different. Instead of making a new game I ported an existing one ([Silt][]) to +Common Lisp. + +I once read somewhere that when trying to build things and learn programming +languages you should either build something you know in a language you're +learning, or build something new in a language you already know, but *not* try +to do both at the same time. I've been getting into Common Lisp over the past +year, so for this game jam I decided to port my [Ludum Dare 34 game][] from +Clojure to Common Lisp. + +The game jam was ten days long. I didn't work on the game every day, but I did +manage to finish porting it over. I improved and polished a few mechanics along +the way, learned a lot, and ended up with a nice little library that sprung out +of the code. I'm happy with the result. + +The code is [on Bitbucket][Silt 2]. You can play the game over telnet if you +want to try it out: `telnet silt.stevelosh.com`. In this post I'm just going to +jot down a few things I found interesting. + +Disclaimer: I'm going to simplify some of the code snippets to make them easier +to read. If you want the full details you can read the actual code. + +[August 2016 Lisp Game Jam]: https://itch.io/jam/august-2016-lisp-game-jam +[Ludum Dare 34 game]: /blog/2015/12/ludum-dare-34/ +[Silt]: http://bitbucket.org/sjl/silt/ +[Silt 2]: http://bitbucket.org/sjl/silt2/ + +{{% toc %}} + + +## Development + +[Silt 2][] is written in Common Lisp. It uses [cl-charms][] (a wrapper around +[ncurses][]) to handle drawing to the terminal, and a few other Common Lisp +libraries like [iterate][] and [cl-arrows][]. + +I developed it on [SBCL][] and OS X, and the telnet server is running Debian so +it works there too. It almost runs in [ClozureCL][], but something +Unicode-related is broken with ncurses under CCL and I didn't bother debugging +it. + +I used [Roswell][] to build a standalone binary for "releases". This binary +starts up much faster than loading everything from scratch. + +I use [Neovim][] and was pleasantly surprised when running ncurses inside +Neovim's terminal emulator Just Worked (especially since the cl-charms `README` +specifically says you *can't* run it in emacs' terminal!). It was really nice to +have the actual game running inside my text editor. + +[iterate]: https://common-lisp.net/project/iterate/ +[cl-arrows]: https://github.com/nightfly19/cl-arrows +[cl-charms]: https://github.com/HiTECNOLOGYs/cl-charms +[ncurses]: https://en.wikipedia.org/wiki/Ncurses +[SBCL]: http://www.sbcl.org/ +[ClozureCL]: http://ccl.clozure.com/ +[Roswell]: https://github.com/roswell/roswell +[Neovim]: https://neovim.io/ + +## ncurses and cl-charms + +[cl-charms][] is a wrapper around [ncurses][] that I used to handle drawing the +game to the terminal. The original Clojure version used [clojure-lanterna][]. + +The game's drawing code is pretty simple, so there's not a whole lot to say +here. I loop over the screen, drawing the contents of each world coordinate at +each screen coordinate, and refresh the window. + +cl-charms mostly worked out great. It's a bit wordy at times (always having to +pass `charms:*standard-window*` to everything), but you can wrap it up pretty +easily. I'd recommend it if you need to do console drawing in Common Lisp. + +cl-charms has a low-level interface that's just an FFI wrapper around ncurses, +and a high-level interface that abstracts some of the Cishness away for you. +I mostly used the high-level interface, but one big thing that's missing is +support for colors. Working with colors in ncurses is a bit tedious, but this +is Lisp so I can just abstract away all the boring stuff: + +```text +(defmacro defcolors (&rest colors) + `(progn + ,@(iterate (for n :from 0) + (for (constant nil nil) :in colors) + (collect `(define-constant ,constant ,n))) + (defun init-colors () + ,@(iterate + (for (constant fg bg) :in colors) + (collect `(charms/ll:init-pair ,constant ,fg ,bg)))))) + +(defcolors + (+color-white-black+ charms/ll:COLOR_WHITE charms/ll:COLOR_BLACK) + (+color-blue-black+ charms/ll:COLOR_BLUE charms/ll:COLOR_BLACK) + (+color-cyan-black+ charms/ll:COLOR_CYAN charms/ll:COLOR_BLACK) + (+color-yellow-black+ charms/ll:COLOR_YELLOW charms/ll:COLOR_BLACK) + (+color-green-black+ charms/ll:COLOR_GREEN charms/ll:COLOR_BLACK) + (+color-pink-black+ charms/ll:COLOR_MAGENTA charms/ll:COLOR_BLACK) + + (+color-black-white+ charms/ll:COLOR_BLACK charms/ll:COLOR_WHITE) + (+color-black-yellow+ charms/ll:COLOR_BLACK charms/ll:COLOR_YELLOW) + + (+color-white-blue+ charms/ll:COLOR_WHITE charms/ll:COLOR_BLUE) + + (+color-white-red+ charms/ll:COLOR_WHITE charms/ll:COLOR_RED) + + (+color-white-green+ charms/ll:COLOR_WHITE charms/ll:COLOR_GREEN)) + +(defmacro with-color (color &body body) + (once-only (color) + `(unwind-protect + (progn + (charms/ll:attron (charms/ll:color-pair ,color)) + ,@body) + (charms/ll:attroff (charms/ll:color-pair ,color))))) + +``` + +[clojure-lanterna]: http://sjl.bitbucket.org/clojure-lanterna/ + +## Using a State Machine as the Game Loop + +One thing many games have in common is a [game loop][]. The original version of +Silt had one, but for the rewrite I decided to structure the main flow of the +game as a state machine instead. This worked out really well and I'm glad I did +it. + +At first I looked around and tried to find a state machine library for Common +Lisp, but then I realized I was being ridiculous and could just model a state +machine with vanilla Lisp functions: + +```text +(defun state-title () + (render-title) + (press-any-key) + (state-intro)) + +(defun state-intro () + (render-intro) + (press-any-key) + (state-generate)) + +(defun state-generate () + (render-generate) + (reset-world) + (generate-world) + (state-map)) + +(defun state-map () + (charms:enable-non-blocking-mode charms:*standard-window*) + (state-map-loop)) + +(defun state-map-loop () + (case (handle-input-map) + ((:quit) (state-quit)) + ((:regen) (state-generate)) + ((:help) (state-help)) + (t (progn + (unless *paused* + (iterate (repeat *frame-skip*) + (tick-world) + (tick-log))) + (render-map) + (when *sleep* + (sleep 0.05)) + (state-map-loop))))) + +(defun state-help () + (render-help) + (press-any-key) + (state-map)) + +(defun state-quit () + 'goodbye) +``` + +This worked especially well with cl-charms and ncurses because for states like +the title and help screens there's no point in looping to redraw the screen over +and over again while waiting for input. I just flipped ncurses into +block-while-awaiting-input mode and let it free up the CPU while waiting for the +user to continue. + +In hindsight I probably should have split out the pause state into a separate +state, which would have let me use blocking input there too. + +Using functions for states like this is only possible because SBCL (and CCL) +perform [last call optimization][], so the stack doesn't get blown by all the +recursion happening. + +[game loop]: https://en.wikipedia.org/wiki/Game_programming#Game_structure +[last call optimization]: https://en.wikipedia.org/wiki/Tail_call + +## Terrain Generation + +The original Silt was made for Ludum Dare 34 in 72 hours, so I didn't spend too +much time on terrain. I just created an empty world and scattered some lakes +around it, which looked like this: + +[![Screenshot of terrain in the original game](/media/images/blog/2016/08/silt1-terrain.png)](/media/images/blog/2016/08/silt1-terrain.png) + +This worked and was quick, but is pretty boring and ugly. In the past few +months I've learned a lot more about terrain generation, so I fleshed things out +a bit more for the new port: + +[![Screenshot of terrain in the new version](/media/images/blog/2016/08/silt2-terrain.png)](/media/images/blog/2016/08/silt2-terrain.png) + +Now I've got oceans and mountains for the creatures to explore. + +### Tiling Diamond Square + +My initial impulse was to use [Perlin Noise][] or [Simplex Noise][] to generate +the heightmap for the world, but I ran into a problem. I wanted the world to be +a torus, just like in the original game, so I needed a terrain generation +algorithm that would generate tileable/wrappable heightmaps. + +One way to do this is to use higher-dimensional noise to get 2D noise that +tiles. If you want to get a 2D heightmap that's tileable in one direction, you +can use 3D noise and take a cylindrical slice of it. To get a heightmap that +tiles both ways you need to use 4D noise. [This article][ron-noise] gives +a really nice overview of the process. + +Unfortunately I couldn't find an implementation of 4D Simplex Noise in Common +Lisp. [black-tie][] and [noise][] both only offer up to 3D noise, and I don't +feel confident enough to implement it myself, even after skimming the simplex +noise paper. + +So I decided to try a different approach and figure out how to modify [Diamond +Square][] to tile. The [Wikipedia article for Diamond Square][ds-wiki] says: + +> Another option [for the diamond step] is to 'wrap around', taking the fourth +> value from the other side of the array. When used with consistent initial +> corner values this method also allows generated fractals to be stitched +> together without discontinuities. + +This sounded great, but after thinking about it for a bit it's obviously not +correct. If we have a heightmap and do what the article says, it will seem to +work at first: + +
+                      ╔══════════════════╗
+    ┌─┬─┬─┬─┬─┐       ║   ┌─┬─┬─┬─┬─┐    ║
+    │5│ │ │ │5│       ║   │5│ │ │ │5│    ║
+    ├─┼─┼─┼─╱┬╲       ║   ├─┼─┼─┼─╱┬╲    ║
+    │ │ │ │╱│││╲      ║   │ │ │ │╱│││╲   ║
+    ├─┼─┼─╱─┼▼┤ ╲     ║   ├─┼─┼─╱─┼▼┤ ╲  ║
+    │ │ │3├─▶◉◀──?    ╚═══════│3├─▶◉◀════╝
+    ├─┼─┼─╲─┼▲┤ ╱         ├─┼─┼─╲─┼▲┤ ╱
+    │ │ │ │╲│││╱          │ │ │ │╲│││╱
+    ├─┼─┼─┼─╲┴╱           ├─┼─┼─┼─╲┴╱
+    │5│ │ │ │5│           │5│ │ │ │5│
+    └─┴─┴─┴─┴─┘           └─┴─┴─┴─┴─┘
+
+ +Wrapping like this will indeed make sure that the averages match up, but there's +two problems. + +First: the corners are all the same value, which means that when you put four +heightmaps next to each other there's an unnatural flat area of four identical +height values next to each other. This probably wouldn't be noticeable in +practice, but if you want to do things *right* it won't be acceptable. + +But the *real* problem is the jitter. If the jitter on one side of the map +happens to be large and positive and the jitter on the other side happens to be +large and negative, you'll get a jarring "cliff" when you try to tile them: + +[![Example of poorly-tiling diamond square](/media/images/blog/2016/08/bad-tiling-ds.png)](/media/images/blog/2016/08/bad-tiling-ds.png) + +The solution I came up with is to reduce the size of the heightmap by 1. +Instead of the heightmap being \\(2^n + 1\\) in each dimension we can make it +\\(2^n\\) and adjust the coordinate-wrapping function appropriately. +Importantly, we *don't* change the calculation of the radius values as we +iterate over the array, so this means quite often we'll be "reaching" for that +final row/column: + +
+     ?       ?
+    ┌─╲─┬─┬─╱
+    │ │╲│ │╱│
+    ├─┼─◢─◣─┤
+    │ │ │◉│ │
+    ├─┼─◥─◤─┤
+    │ │╱│ │╲│
+    ├─╱─┼─┼─╲
+    │5│ │ │ │?
+    └─┴─┴─┴─┘
+
+ +When we try to access that nonexistent coordinate we just wrap around back to +zero. Notice that we also only need to initialize a single corner cell now. + +It's a simple change, but the result is *much* nicer: + +[![Example of nicely-tiling diamond square](/media/images/blog/2016/08/good-tiling-ds.png)](/media/images/blog/2016/08/good-tiling-ds.png) + +[Perlin Noise]: https://en.wikipedia.org/wiki/Perlin_noise +[Simplex Noise]: https://en.wikipedia.org/wiki/Simplex_noise +[ron-noise]: http://ronvalstar.nl/creating-tileable-noise-maps +[black-tie]: https://github.com/aerique/black-tie +[noise]: https://github.com/sebity/noise +[Diamond Square]: /blog/2016/06/diamond-square/ +[ds-wiki]: https://en.wikipedia.org/wiki/Diamond-square_algorithm + +## Entity, Aspects, and Systems + +Terrain generation is pretty, but the next step in the port was to add some +plants, creatures, and artifacts. In the original game I just represented +things in the world as vanilla Clojure maps, but that was getting kind of messy +and I wanted to try a different approach this time. + +Recently I read through [Game Engine Architecture][] (a *fantastic* book) and +made a few games in [Unity][], which together made me want to try using an +[Entity/Component System][] this time around. There are a couple of ECS +libraries out there for Common Lisp like [cl-ecs][] and [ecstasy][], but in true +Lisp fashion I ended up not being quite satisfied with any of them and writing +Yet Another God Damn Library. + +It's called [Beast][]. It's subtly different than the others in that it prefers +to be a really thin layer over CLOS and uses inheritance instead of composition. +It uses the word "aspect" instead of "component" to try to overload that word +a bit less, so it's the "Basic Entity/Aspect/System Toolkit". It ended up being +about 150 lines of code (not including docstrings), so I managed to avoid going +down too much of a rabbit hole during the jam. + +If you want to know all the details, check out its documentation (it has +*actual* documentation). But here I'll just talk about a couple of the +particular bits of Silt that I used it for. + +[Unity]: https://unity3d.com/ +[Game Engine Architecture]: http://www.amazon.com/dp/1466560010/?tag=stelos-20 +[Entity/Component System]: https://en.wikipedia.org/wiki/Entity_component_system +[cl-ecs]: https://github.com/lispgames/cl-ecs +[ecstasy]: https://github.com/mfiano/ecstasy +[beast]: http://sjl.bitbucket.org/beast/overview/ + +### Coordinates + +The first thing I needed was a way to keep track of where things are in the +world. + +If the world space were continuous a [quadtree][] would have been my first +choice, but in Silt the world is split into discrete integer coordinates. +Creatures move directly from \\((x, y)\\) to \\((x+1, y+1)\\). I decided to use +a simple array of lists to represent this: + +```text +(defparameter *coords-contents* + (make-array (list +world-size+ +world-size+) + :initial-element nil)) +``` + +Each value in the array is a list of the entities that are currently there. +This means looking up what things are at a given coordinate is a single fast +`aref`. + +I tried using a hash table instead of an array at first, thinking that if the +world were fairly sparse it would be wasteful to allocate an array with a ton +of `nil` values in it. But the array method is much faster for looking things +up (which happens a lot) and memory is cheap, so I decided against the hash +tables. It worked great in the end. + +Entities need to know where they are in the world, so I defined a Beast aspect +for that: + +```text +(define-aspect coords x y) +``` + +Then I defined a few functions to handle moving entities into, out of, and +around the world: + +```text +(defun coords-insert-entity (e) + (push e (aref *coords-contents* (coords/x e) (coords/y e)))) + +(defun coords-remove-entity (e) + (zap% (aref *coords-contents* (coords/x e) (coords/y e)) + #'delete e %)) + +(defun coords-move-entity (e new-x new-y) + (coords-remove-entity e) + (setf (coords/x e) (wrap new-x) + (coords/y e) (wrap new-y)) + (coords-insert-entity e)) + +(defun coords-lookup (x y) + (aref *coords-contents* (wrap x) (wrap y))) +``` + +Entities might also like to know what's near them: + +```text +(defun nearby (entity &optional (radius 1)) + (remove entity + (iterate + outer + (with x = (coords/x entity)) + (with y = (coords/y entity)) + (for dx :from (- radius) :to radius) + (iterate + (for dy :from (- radius) :to radius) + (in outer + (appending (coords-lookup (+ x dx) + (+ y dy)))))))) +``` + +This ends up compiling down to a nice tight loop of \\((2 * radius + 1)^2\\) +`aref`s. I only wish iterate had a nicer syntax for looping over nested +indices like this. I'm sure it's possible to write an iterate driver for it -- +maybe someday I'll try making one. + +I also needed a way to get entities into the world array when they're created +and remove them when they die. Beast (well, actually CLOS) makes this trivially +easy with auxiliary methods: + +```text +(defmethod entity-created :after ((entity coords)) + (coords-insert-entity entity)) + +(defmethod entity-destroyed :after ((entity coords)) + (coords-remove-entity entity)) +``` + +[quadtree]: https://en.wikipedia.org/wiki/Quadtree + +### User Interface + +Once I had a way of know where things are, the next step was to display them on +the screen. I broke this into a few separate aspects. + +#### Visible + +The `visible` aspect is for things that are drawn on the screen with +a particular glyph and color: + +```text +(define-aspect visible glyph color) + +;; ... + +(define-entity tree (coords visible ...)) + +(defun make-tree (x y) + (create-entity 'tree + :coords/x x + :coords/y y + :visible/glyph "T" + :visible/color +color-green-black+ + ;; ... + )) + +``` + +The drawing code can then figure out what to draw for each screen coordinate: + +```text +(defun draw-map () + (iterate + (for sx :from 0 :below *screen-width*) + (for wx :from *view-x*) + (iterate + (for sy :from 0 :below *screen-height*) + (for wy :from *view-y*) + (for entity = (find-if #'visible? (coords-lookup wx wy))) + (if entity + (with-color (visible/color entity) + (write-string-at (visible/glyph entity) sx sy)) + ;;; otherwise draw the terrain + (...))))) +``` + +Again: my kingdom for a `(for-nested ...)` iterate driver! But the core is just +using `(find-if #'visible? (coords-lookup wx wy))` to find the first visible +thing and then drawing it: + +[![Screenshot of entities with the visible aspect](/media/images/blog/2016/08/aspect-visible.png)](/media/images/blog/2016/08/aspect-visible.png) + +I used `find-if` instead of `remove-if-not` because we can only draw one +character to a given position in the terminal anyway, so I just pick the first +thing that happens to be in the list. + +#### Flavor + +The `flavor` aspect is for adding [flavor text][] that appears when the user +puts their cursor over an entity: + +```text +(define-aspect flavor text) + +;; ... + +(define-entity tree (coords visible flavor ...)) + +(defun make-tree (x y) + (create-entity 'tree + :coords/x x + :coords/y y + :visible/glyph "T" + :visible/color +color-green-black+ + :flavor/text + '("A tree sways gently in the wind."))) +``` + +Then when the user's cursor is at a certain position I can find all the entities +there and draw the flavor text for any that have the `flavor` aspect: + +```text +(defun draw-selected () + (write-left + (iterate + (for entity :in (multiple-value-call #'coords-lookup + (screen-to-world *cursor-x* *cursor-y*))) + (when (typep entity 'flavor) + (appending (flavor/text entity) :into text) + + ;; ... + + (collecting "" :into text)) + (finally (return text))) + 1 1 :pad t)) +``` + +Which looks like this: + +[![Screenshot of flavor text](/media/images/blog/2016/08/aspect-flavor.png)](/media/images/blog/2016/08/aspect-flavor.png) + +Of course the flavor text doesn't have to be a constant: + +```text +(defun make-creature (x y &key + (color +color-white-black+) + (glyph "@")) + (let ((name (random-name))) + (create-entity 'creature + :name name + :coords/x x + :coords/y y + :visible/color color + :visible/glyph glyph + :flavor/text + (list (format nil "A creature named ~A is here." name) + "It likes food.")))) +``` + +[flavor text]: https://en.wikipedia.org/wiki/Flavor_text + +#### Inspectable + +The last thing I wanted was an easy way to show attributes of entities in the +main game UI. The original Clojure game just dumped the entire object to the +screen: + +[![Screenshot of creature inspection in the original game](/media/images/blog/2016/08/silt1-inspect.png)](/media/images/blog/2016/08/silt1-inspect.png) + +But this time I wanted a bit more control. The `inspectable` aspect has a list +of things that should be displayed. These can be symbols (which denote CLOS +slot names) or functions that return `(label . text)` conses: + +```text +(define-aspect inspectable + (slots :initform nil)) + +(defun inspectable-get (entity slot) + (etypecase slot + (symbol (cons slot (slot-value entity slot))) + (function (funcall slot entity)))) +``` + +When creating an entity I can just list out the slots I want to be displayed on +the screen, or use a little `lambda` if I want to show something that's not an +actual slot: + +```text +(defun make-fruit (x y) + (create-entity 'fruit + ;; ... + :inspectable/slots '(edible/energy))) + +(defun make-creature (x y &key ...) + (let ((name (random-name))) + (create-entity 'creature + ;; ... + :inspectable/slots + (list 'name + (lambda (c) (cons 'directions ...)) + 'metabolizing/energy + 'metabolizing/insulation + 'aging/birthtick + 'aging/age)))) +``` + +Then I just append some extra text for `inspectable` entities when drawing +descriptions of things at the cursor position: + +```text +(defun draw-selected () + (write-left + (iterate + (for entity :in (multiple-value-call #'coords-lookup + (screen-to-world *cursor-x* *cursor-y*))) + (when (typep entity 'flavor) + ;; ... + (when (typep entity 'inspectable) + (appending + (indent + (iterate + (with slots = (mapcar (curry #'inspectable-get entity) + (inspectable/slots entity))) + (with width = (apply #'max + (mapcar (compose #'length #'symbol-name #'car) + slots))) + (for (label . contents) :in slots) + (collect + (let ((*print-pretty* nil)) + (format nil "~vA ~A" width label contents))))) + :into text)) + + (collecting "" :into text)) + (finally (return text))) + 1 1 :pad t)) +``` + +This is pretty ugly because I wanted to justify and indent things nicely, but +the result looks much nicer than the original game: + +[![Screenshot of creature inspection in the new version](/media/images/blog/2016/08/silt2-inspect.png)](/media/images/blog/2016/08/silt2-inspect.png) + +### Food + +Seeing the world is nice, but we also want the things in it to actually *do* +something. The world revolves heavily around food and energy, so I defined +a few aspects to handle things: + +```text +(define-aspect edible + energy + original-energy) + +(define-aspect decomposing + rate + (remaining :initform 1.0)) + +(define-aspect fruiting + chance) + +(defmethod initialize-instance :after ((e edible) &key) + (setf (edible/original-energy e) + (edible/energy e))) +``` + +I do wish there was a slightly less wordy way to default the value of one slot +to another one, but oh well. + +Then I just added the aspects to the appropriate entities: + +```text +(define-entity tree (coords visible fruiting flavor)) +(define-entity fruit (coords visible edible flavor decomposing inspectable)) +(define-entity algae (coords visible edible decomposing)) +(define-entity grass (coords visible edible decomposing)) +(define-entity corpse (coords visible flavor decomposing)) +``` + +Trees can grow fruit, so they have the `fruiting` aspect. The `grow-fruit` +Beast system handles growing some each tick: + +```text +(define-system grow-fruit ((entity fruiting coords)) + (when (randomp (fruiting/chance entity)) + (make-fruit (wrap (random-around (coords/x entity) 2)) + (wrap (random-around (coords/y entity) 2))))) +``` + +Fruit is `edible`, but also decomposes over time. It's got `flavor` and +`inspectable` aspects so you can see how much energy is left. + +I added algae and grass as secondary food sources to spread out the food supply +a bit more and make the creatures a bit less dependent on the trees. I didn't +give these flavor text to avoid cluttering up the UI too much. + +I considered making corpses edible too, but figured that might be a bit too +gruesome. So corpses decompose, but the critters aren't cannibals. + +I made a couple of Beast systems to handle the process of decomposing things +every game tick: + +```text +(define-system rot ((entity decomposing)) + (when (minusp (decf (decomposing/remaining entity) + (decomposing/rate entity))) + (destroy-entity entity))) + +(define-system rot-food ((entity decomposing edible)) + (setf (edible/energy entity) + (lerp 0.0 (edible/original-energy entity) + (decomposing/remaining entity)))) +``` + +`rot` runs on everything with the `decomposing` aspect. It ticks along the +progress of an entity's decomposition, and destroys it once it's finished. + +`rot-food` runs on every entity that's both `decomposing` *and* `edible`. It +reduces the energy value of the food over time, because rotten food is less +healthy. I'm pretty happy with how easy Beast makes this kind of thing. + +### Creatures and Mysteries + +The final pieces of the world are the creatures and artifacts. + +#### Energy +Creatures need food (energy) to survive. I modeled this with a `metabolizing` +aspect and `consume-energy` system: + +```text +(define-aspect metabolizing insulation energy) + +(defmethod starve ((entity entity)) + (destroy-entity entity)) + +(defmethod calculate-energy-cost ((entity metabolizing)) + (let* ((insulation (metabolizing/insulation entity)) + (base-cost 1.0) + (temperature-cost (max 0 (* 0.2 (- (abs *temperature*) insulation)))) + (insulation-cost (* 0.1 insulation))) + (+ base-cost temperature-cost insulation-cost))) + +(define-system consume-energy ((entity metabolizing)) + (when (minusp (decf (metabolizing/energy entity) + (calculate-energy-cost entity))) + (starve entity))) +``` + +I made `starve` and `calculate-energy-cost` generic functions because I thought +I might eventually have different metabolizing things in the world and might +want to override them. I didn't end up doing this in the end (creatures are the +only things that burn energy) so these could have been normal functions. + +The energy mechanic works similarly to the original game: + +* Creatures spend a bit of energy each tick to stay alive. +* When you make the temperature hotter or colder, it costs additional energy per + tick for the creatures to live. +* Creatures sometimes gain/lose insulation during reproduction, which mitigates + the energy cost of the temperature difference. +* Insulation itself costs a little bit of energy every tick. + +The effect is that if you change the temperature gradually over time, the +population will evolve higher insulation values (because the children with more +insulation are more likely to survive longer). If you then set the temperature +back to zero (the ideal) the population will eventually evolve to shed the +insulation, because it costs a little bit of energy and doesn't provide any +benefit when the world is pleasant. Natural selection is fun. + +The last piece of the puzzle is letting things actually take action. Creatures +and some artifacts need to take an action on every tick, while other artifacts +only do things occasionally. A pair of aspects and systems handles the +bookkeeping here: + +```text +(define-aspect sentient function) +(define-aspect periodic + function + (counter :initform 1) + next + min + max) + +(define-system sentient-act ((entity sentient)) + (funcall (sentient/function entity) entity)) + +(define-system periodic-tick ((entity periodic)) + (when (zerop (setf (periodic/counter entity) + (mod (1+ (periodic/counter entity)) + (periodic/next entity)))) + (setf (periodic/next entity) + (random-range (periodic/min entity) + (periodic/max entity))) + (funcall (periodic/function entity) entity))) +``` + +I'm not going to go over all the actual AI and actions, you can take a look at +the code if you're curious. + +## Random Name Generation + +I wanted to add a more personal connection to the creatures this time around, so +I decided they should have names. I used a really simple form of +[syllable-based name generation][namegen] to give each creature its own random +name: + +```text +(defparameter *name-syllables* + (-> "syllables.txt" + slurp + read-from-string + (coerce 'vector))) + +(defun random-name () + (format nil "~:(~{~A~}~)" + (iterate (repeat (random-range 1 5)) + (collect (random-elt *name-syllables*))))) +``` + +To get a random name I just smash together one to four random syllables. To +make a list of syllables I grabbed some Icelandic text and made a pair of really +janky shell and Python scripts to print out every 3/4/5-letter chunk of every +word, sort them by frequency, and take the top 500. + +They're not *really* syllables but they're okay for just a couple of lines of +code and a few minutes work: + +[![Screenshot of creature names](/media/images/blog/2016/08/silt-names.png)](/media/images/blog/2016/08/silt-names.png) + +[namegen]: http://www.roguebasin.com/index.php?title=Syllable-based_name_generation + +## Simple Data Structures + +As I coded things up I wound up with a handy pair of data structures I might use +again for other things in the future. + +### Weightlists + +Each game tick a creature needs to decide which direction to walk. At the start +of the game they just pick a random direction, but as they reproduce their +children can mutate to prefer certain directions over others. + +The natural selection of Silt turns out to prefer creatures that wander around +to those that stay in place. Fruit takes time to grow, so it's more effective +to travel around and gather it than to sit in place waiting for it to regrow. + +The original game just used a Clojure vector of weights and directions to +represent how much a creature prefers each direction. That worked, but the +weights are only ever set/changed when a creature is born, and a random element +is chosen every turn. It's more efficient in the long run if we precompute +a few things up front, so I made a little "weightlist" API: + +```text +(defstruct (weightlist (:constructor %make-weightlist)) + weights sums items total) + +(defun make-weightlist (items weights) + "Make a weightlist of the given items and weights." + (%make-weightlist + :items items + :weights weights + :sums (prefix-sums weights) + :total (apply #'+ weights))) + +(defun weightlist-random (weightlist) + "Return a random item from the weightlist, taking the weights into account." + (iterate + (with n = (random (weightlist-total weightlist))) + (for item :in (weightlist-items weightlist)) + (for weight :in (weightlist-sums weightlist)) + (finding item :such-that (< n weight)))) +``` + +This is pretty straightforward. Note that the weights can be integers or floats +(or some of each!) and things will Just Work, because Common Lisp's `random` can +take either. Weights of zero are fine too, as long as at least one element has +a nonzero weight. + +### Ticklists + +In a couple of places I needed some kind of list where items in it expire over +time. For example: + +* The Fountain artifact only lets creatures drink from it once every thousand + ticks, so I needed a way to keep track of the entities that had drank + recently. +* The game log at the bottom of the screen contains messages that should be + shown for a certain number of ticks, then disappear. + +I made a simple little thing I called a "ticklist" to handle these: + +```text +(defun make-ticklist () + nil) + +(defmacro ticklist-push (ticklist value lifespan) + `(push (cons ,lifespan ,value) ,ticklist)) + +(defun ticklist-tick (ticklist) + (flet ((decrement (entry) + (decf (car entry))) + (dead (entry) + (minusp (car entry)))) + (->> ticklist + (mapc #'decrement) + (remove-if #'dead)))) + +(defun ticklist-contents (ticklist) + (mapcar #'cdr ticklist)) +``` + +Internally a ticklist is just a list of `(remaining-ticks . thing)` conses, but +the rest of my code doesn't have to care about that: + +```text +(defun fountain-act (f) + (with-slots (recent) f + (zapf recent #'ticklist-tick) + (iterate + (with already-drank = (ticklist-contents recent)) + (for creature :in (remove-if-not #'creature? (nearby f))) + (unless (member creature already-drank) + (creature-mutate-appearance creature) + (ticklist-push recent creature 1000) + (log-message "~A drinks from the fountain and... changes." + (creature-name creature)))))) + +(defun log-message (s &rest args) + (ticklist-push *game-log* (apply #'format nil s args) 200)) + +(defun state-map-loop () + ;; ... + (unless *paused* + (iterate (repeat *frame-skip*) + (tick-world) + (zapf *game-log* #'ticklist-tick)))) +``` + +## Profiling and Performance + +When something starts slowing things down it's helpful to be able to turn on +profiling and see what's going on. SBCL has a nice statistical profiler, so +I made a couple of functions to flip it on and off as needed: + +```text +#+sbcl +(defun dump-profile () + (with-open-file (*standard-output* "silt.prof" + :direction :output + :if-exists :supersede) + (sb-sprof:report :type :graph + :sort-by :cumulative-samples + :sort-order :ascending) + (sb-sprof:report :type :flat + :min-percent 0.5))) + +#+sbcl +(defun start-profiling () + (sb-sprof::reset) + (sb-sprof::profile-call-counts "SILT") + (sb-sprof::start-profiling :max-samples 50000 + ; :mode :cpu + :mode :time + :sample-interval 0.01 + :threads :all)) + +#+sbcl +(defun stop-profiling () + (sb-sprof::stop-profiling) + (dump-profile)) +``` + +When I wanted to check performance I could just evaluate `(start-profiling)` +over NREPL and let the game continue to run, then `(stop-profiling)` a little +bit later and look at the results. It came in handy once or twice when tracking +down some slowness. + +## Future Improvements and Ideas + +This game jam was quite a bit of fun! I'm happy with the results and feel like +I've learned a lot along the way. + +I'm done with the game and don't plan on updating it any more, but I'll scribble +down a few extra ideas for things that could be improved here, just to get them +out of my head: + +* Figure out the Unicode issues with cl-charms and CCL. +* Contribute to cl-charms to add some higher-level tools for working with color. +* Contribute to one of the Common Lisp noise libraries to implement the 4D + variant of Simplex Noise. +* Flesh out the name generation into something much nicer and more polished. +* Implement different costs for moving over different terrain. +* Add health, fighting, and carnivores. +* Add more mysterious artifacts to the world. +* Flesh out the vegetation model to let trees grow and die, algae spread, etc. +* Improve the visuals. [Brogue][] proves you can do far more than you might + think with just Unicode characters. +* Model senses like vision, providing the creatures with more information but + with an energy cost. +* Give creatures "brains" by generating and mutating actual Lisp code. This + would let the creatures learn strategies over time, though I'm not sure how + feasible it would be. +* Improve performance by profiling much more and fixing the hottest parts of the + code. +* Add saving and loading of the world. +* Add the ability to seed the RNG and make everything deterministic, so people + can share interesting seeds. Doing this for the terrain generation at least + should be pretty easy, the game as a whole might be slightly trickier. +* Improve the UI a bit more, maybe using ncurses' support for windows layered on + top of other windows. + +[Brogue]: https://sites.google.com/site/broguegame/ + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2016/08/playing-with-syntax.html --- a/content/blog/2016/08/playing-with-syntax.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,493 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "Playing With Syntax" - snip: "Lisp lets you evolve your language." - created: 2016-08-19 13:15:00 - %} - - {% block article %} - -One of the things I love about Lisp is that it gives you the ability to change -and mold the syntax of the language to what you need. In this post I want to -look at the evolution of a little macro I've been playing around with for -a while now. - -Mutation is a fundamental concept in most programming languages. Functional -programming may be beautiful, but mutation is still useful (and fast). In most -languages assignment is done with an `=` or `:=` operator: - - :::lisp - x = 10 - -In Common Lisp the operator is named `setf` instead of `=` for historical -reasons, and of course it's used in prefix notation: - - :::lisp - (setf x 10) - -Aside from the prefix ordering, Common Lisp's syntax is already a bit more -elegant because you can set arbitrarily many variables without repeating the -assignment operator over and over again: - - :::lisp - ; Have to type out `=` three times - x = 10 - y = 20 - z = 30 - - ; But only one `setf` required - (setf x 10 - y 20 - z 30) - -`setf` assigns values one-by-one, in-order. Common Lisp also has `psetf` which -is "parallel `setf`". This evaluates all the values first, *then* sets all the -variables at once: - - :::lisp - (let ((a 10) (b 20)) - (setf a 50 - b (+ 1 a)) ; => (50 51) - - (psetf a 90 - b (+ 1 a))) ; => (90 52) - -Assignment is nice, but often you want to change the value of a variable by -taking its *current* value and computing something with it. If you're making -a ball that travels across the screen you might have something like this: - - :::text - def move_ball(ball, distance): - ball.x = ball.x + distance - - (defun move-ball (ball distance) - (setf (slot-value ball 'x) - (+ (slot-value ball 'x) - distance))) - -If you find `slot-value` too wordy to type, you could make a macro to get rid of -it (and save yourself a quote too). `.` is already used for something else so -you'd need to pick a different character: - - :::lisp - (defmacro $ (instance slot-symbol) - `(slot-value ,instance ',slot-symbol)) - - (defun move-ball (ball distance) - (setf ($ ball x) - (+ ($ ball x) - distance))) - -In practice, though, you rarely use `slot-value`. Instead you usually use -accessor functions: - - :::lisp - (defun move-ball (ball distance) - (setf (ball-x ball) - (+ (ball-x ball) - distance))) - -Anyway, back to assignment. What we wrote above *works*, but it's a bit -annoying to have to type the "place" that we're working with twice. To cut down -on this repetition many languages offer operators like `+=` and `++`: - - :::text - x = x + 10 - ⬇ - x += 10 - - y = y + 1 - ⬇ - y++ - -Common Lisp's version of these is called `incf`. - - :::lisp - (incf x 10) - (incf y) - -Notice how the s-expression syntax means we can use the same operator for both -of the forms, instead of needing separate `+=` and `++` operators. - -Other languages often have similar "in-place" operators for other numeric -operations like `-=`, `--`, `*=`, and so on. Common Lisp has `decf` for -subtraction, but doesn't have versions for multiplication or division built in. -But we can add them with a few macros: - - :::lisp - (defmacro mulf (place factor) - `(setf ,place (* ,place ,factor))) - - (defmacro divf (place divisor) - `(setf ,place (/ ,place ,divisor))) - -Unfortunately these are not quite right -- we've committed the cardinal -macro-writing sin of splicing in the `place` expression multiple times. If -`place` has side effects they'll happen more times than expected. - -Luckily, Common Lisp has a macro that wraps up the boring process of handling -this for us. `define-modify-macro` lets us fix the problem: - - :::lisp - (define-modify-macro mulf (factor) *) - (define-modify-macro divf (divisor) /) - -Now we've got versions of `*=` and `/=` in Common Lisp. While we're here we -should do things *right* and allow `divf` to be used with just a place, which -will mean "set `place` to `1/place`": - - :::lisp - (define-modify-macro divf (&optional divisor) - (lambda (value divisor) - (if (null divisor) - (/ 1 value) - (/ value divisor)))) - - (let ((x 10)) - (divf x 2) ; => 5 - (divf x) ; => 1/5 - (divf x)) ; => 5 - -Note that we don't really need the `1` in `(/ 1 value)`, because Common Lisp's -`/` works the same way when you call it with a single argument. I'm not sure -what behavior would make sense for a unary `(mulf place)`, so let's ignore that -one and move on. - -So far we've been talking about the idea of setting a variable to the result of -computing some function on its current value. We used `define-modify-macro` to -define some support for extra functions, but defining a mutator macro for every -function we might possibly want to use could get tedious. What if we generalize -this a bit more? - - :::lisp - (define-modify-macro callf (function) - (lambda (value function) - (funcall function value))) - - (let ((x -1)) - (callf x #'abs) ; => 1 - (callf x #'1+) ; => 2 - (callf x #'sin) ; => 0.9092974 - (callf x #'round) ; => 1 - (callf x #'-)) ; => -1 - -Now we've got something a bit higher-level. With `callf` we can mutate -a variable with a unary function. But what about functions that need more -arguments? Again, Common Lisp does the right thing and handles `&rest` -parameters correctly in `define-modify-macro`: - - :::lisp - (define-modify-macro callf (function &rest args) - (lambda (value function &rest args) - (apply function value args))) - - (let ((x -1)) - (callf x #'abs) ; => 1 - (callf x #'+ 10) ; => 11 - (callf x #'* 2 4)) ; => 88 - - (let ((foo (list 0 1 2 3))) - (callf foo #'reverse) - ; => (3 2 1 0) - - (callf foo #'append (list -1 -2 -3))) - ; => (3 2 1 0 -1 -2 -3) - -That's better. Now we can mutate a variable by applying a function to its -current value and some other arguments. This might come in handy when we don't -want to bother defining a full-blown modification macro just to use it once or -twice. - -At this point we've got something similar to Michael Malis' [zap][] macro. -A minor difference is for `zap` the order of the function and the place are -reversed. I believe his reason was that it makes the resulting form look more -like the function call that is conceptually being made: - - :::lisp - (zap #'+ x 5) - ; ↓ ↓ ↓ - (setf x ( + x 5 )) - -Unfortunately, because the place is no longer the first argument we can't use -`define-modify-macro` to handle the single-evaluation boilerplate for us any -more. Instead we need to use `get-setf-expander` and work with its results. -Malis' excellent [Getting Places][] article explains how that works, and if you -want to fully understand the rest of the code in this post you should read that -one first. - -The difference between `callf` and Malis' `zap` are just cosmetic. But they're -still not completely flexible. What if we want to call a function, but we need -the value to be an argument other than the first? Of course we could use -a `lambda`: - - :::lisp - (let ((x (list :coffee :soda :tea))) - (callf x (lambda (l) (remove :soda l)))) - - -But that is *really* ugly. It would be nice if we had a way to specify which -argument to the function should be the current value. This is Lisp, so we can -pick whatever syntax we want. What should we choose? Often a good first step -is to look at how other languages do things. Clojure's anonymous function -syntax is one possibility: - - :::lisp - (map #(+ 10 %) [1 2 3]) - ; => (11 12 13) - -Clojure uses the magic symbol `%` as a placeholder for the argument. We can -modify Malis' `zap` macro to do the same. We'll call it `zapf`, and we'll swap -the place back into the first argument like all the other modify macros, since -the aesthetic reason no longer holds. The changes are in bold: - -
-; Original version
-(defmacro zap (fn place &rest args)
-  (multiple-value-bind
-        (temps exprs stores store-expr access-expr)
-      (get-setf-expansion place)
-    `(let* (,@(mapcar #'list temps exprs)
-            (,(car stores)
-              (funcall ,fn ,access-expr ,@args)))
-       ,store-expr)))
-
-; New version
-(defmacro zapf (place fn &rest args)
-  (multiple-value-bind
-        (temps exprs stores store-expr access-expr)
-      (get-setf-expansion place)
-    `(let* (,@(mapcar #'list temps exprs)
-            (,(car stores)
-             (funcall ,fn ,@(substitute access-expr '% args))))
-      ,store-expr)))
-
- -And now we can use `%` to say where the place's value should go: - - :::lisp - (let ((x (list :coffee :soda :tea))) - (zapf x #'remove :soda %) - ; => (:coffee :tea) - - (zapf x #'cons :water %) - ; => (:water :coffee :tea) - - (zapf x #'append % (list :cocoa :chai))) - ; => (:water :coffee :tea :cocoa :chai) - -Note that `substitute` will replace *all* occurrences of `%` with the access -expression, so we can do something like `(zapf x #'append % %)` if we want. - -Some people will find this new version unpleasant, because it effectively -captures the `%` variable. It's not hygienic, by design. I personally don't -mind it, and I like the brevity it allows. - -We're almost done, but I want to push things just a *bit* further. Let's -revisit the original example of the ball moving across the screen: - - :::text - def move_ball(ball, distance): - ball.x = ball.x + distance - - (defun move-ball (ball distance) - (setf (ball-x ball) - (+ (ball-x ball) distance))) - -Of course we can simply use `+=` or `incf` here: - - :::text - def move_ball(ball, distance): - ball.x += distance - - (defun move-ball (ball distance) - (incf (ball-x ball) distance)) - -Suppose we also wanted to make the ball wrap around the screen when it flies off -one side. We can do this using modulus: - - :::text - def move_ball(ball, distance, screen_width): - ball.x += distance - ball.x %= screen_width - - (defun move-ball (ball distance screen-width) - (incf (ball-x ball) distance) - (callf (ball-x ball) #'mod % screen-width)) - -We could define `modf` if we wanted to make that second call a bit nicer. But -let's take a moment to reflect. Notice how we're back to having to type out -`(ball-x ball)` twice again. It's better than typing it four times, but can we -do better? - -`zapf` currently takes a place, a function, and a list of arguments. What if we -made it more like `setf`, and just have it take a place and an expression? We -could still use `%` as a placeholder, but could let it appear anywhere in the -(possibly nested) expression form. - -One way to do this would be to simply replace `substitute` with `subst` in the -`zapf` code. (The difference between these extremely-confusingly-named -functions is that `substitute` replaces element of a sequence, and `subst` -replaces elements of a tree.) - -This, however, is the wrong strategy. Blindly replacing `%` everywhere in the -expression will work for many cases, but will break in edge cases like (god help -you) nested `zapf` forms. We could try to walk the code of the expression we -get, but down this path lies madness and implementation-specificity. - -The solution is much, much simpler. We can just use a normal `let` to capture -`%` in the expression: - -
-; Final version
-(defmacro zapf (place expr)
-  (multiple-value-bind
-        (temps exprs stores store-expr access-expr)
-      (get-setf-expansion place)
-    `(let* (,@(mapcar #'list temps exprs)
-            (,(car stores)
-             (let ((% ,access-expr))
-               ,expr)))
-      ,store-expr)))
-
- -And now `move-ball` only requires the bare minimum of typing: - - :::lisp - (defun move-ball (ball distance screen-width) - (zapf (ball-x ball) - (mod (+ distance %) screen-width))) - -For completeness it would be nice to allow `zapf` to take any number of -place/value pairs, just like `setf`. I'll leave this as an exercise for you to -try if you're interested. - -It took me a while to arrive at this form of the macro. I personally find it -lovely and readable. If you disagree, that's fine too! Lisp is a wonderful -substrate for building a language that fits how you think and talk about -problems. Play around with your own syntax and find out what feels most natural -for you. - -Before we leave, let's just ponder one more thing. Performance is often ignored -these days, but what if we care about making the computer go fast? The final -`zapf` macro is handy and expressive, but what cost have we paid for this -abstraction? - -First let's tell SBCL that we want to go fast and return to our original `setf` -strategy: - - :::lisp - (deftype nonnegative-fixnum () - `(integer 0 ,most-positive-fixnum)) - - (deftype positive-fixnum () - `(integer 1 ,most-positive-fixnum)) - - (defstruct ball - (x 0 :type nonnegative-fixnum) - (y 0 :type nonnegative-fixnum)) - - (declaim (ftype (function (ball fixnum positive-fixnum)) - move-ball)) - - (defun move-ball (ball distance screen-width) - (declare (optimize (speed 3) (safety 0) (debug 0))) - (setf (ball-x ball) - (mod (+ distance (ball-x ball)) - screen-width))) - -How does that turn out? - - :::text - ; disassembly for MOVE-BALL - ; Size: 82 bytes. Origin: #x1005E5E12E - ; 2E: 488B410D MOV RAX, [RCX+13] ; no-arg-parsing entry point - ; 32: 48D1F8 SAR RAX, 1 - ; 35: 498D3C00 LEA RDI, [R8+RAX] - ; 39: 488BDE MOV RBX, RSI - ; 3C: 48D1FB SAR RBX, 1 - ; 3F: 4885DB TEST RBX, RBX - ; 42: 7431 JEQ L3 - ; 44: 488BC7 MOV RAX, RDI - ; 47: 4899 CQO - ; 49: 48F7FB IDIV RAX, RBX - ; 4C: 4C8BC0 MOV R8, RAX - ; 4F: 488BC2 MOV RAX, RDX - ; 52: 48D1E0 SHL RAX, 1 - ; 55: 4885C0 TEST RAX, RAX - ; 58: 7510 JNE L2 - ; 5A: L0: 488BD8 MOV RBX, RAX - ; 5D: L1: 4889590D MOV [RCX+13], RBX - ; 61: 488BD3 MOV RDX, RBX - ; 64: 488BE5 MOV RSP, RBP - ; 67: F8 CLC - ; 68: 5D POP RBP - ; 69: C3 RET - ; 6A: L2: 4885FF TEST RDI, RDI - ; 6D: 7DEB JNL L0 - ; 6F: 488D1C30 LEA RBX, [RAX+RSI] - ; 73: EBE8 JMP L1 - ; 75: L3: 0F0B0A BREAK 10 ; error trap - ; 78: 07 BYTE #X07 - ; 79: 08 BYTE #X08 ; DIVISION-BY-ZERO-ERROR - ; 7A: FE9E03 BYTE #XFE, #X9E, #X03 ; RDI - ; 7D: FE9E01 BYTE #XFE, #X9E, #X01 ; RBX - -Well that's not *too* bad. I'm not sure why SBCL still performs a check for -division by zero for the `mod` even though we said that `screen-width` can't -possibly be zero. But hey, we're at a manageable amount of assembly, which is -pretty nice for a high-level language! - -Now for the `zapf` version: - - :::lisp - (defun move-ball (ball distance screen-width) - (declare (optimize (speed 3) (safety 0) (debug 0))) - (zapf (ball-x ball) - (mod (+ distance %) screen-width))) - -Okay, let's bite the bullet. How bad are we talking? - - :::text - ; disassembly for MOVE-BALL - ; Size: 70 bytes. Origin: #x1005F47C7B - ; 7B: 488B410D MOV RAX, [RCX+13] ; no-arg-parsing entry point - ; 7F: 48D1F8 SAR RAX, 1 - ; 82: 488D3C03 LEA RDI, [RBX+RAX] - ; 86: 4885F6 TEST RSI, RSI - ; 89: 742B JEQ L2 - ; 8B: 488BC7 MOV RAX, RDI - ; 8E: 4899 CQO - ; 90: 48F7FE IDIV RAX, RSI - ; 93: 488BD8 MOV RBX, RAX - ; 96: 488D0412 LEA RAX, [RDX+RDX] - ; 9A: 4885C0 TEST RAX, RAX - ; 9D: 750D JNE L1 - ; 9F: L0: 48D1E2 SHL RDX, 1 - ; A2: 4889510D MOV [RCX+13], RDX - ; A6: 488BE5 MOV RSP, RBP - ; A9: F8 CLC - ; AA: 5D POP RBP - ; AB: C3 RET - ; AC: L1: 4885FF TEST RDI, RDI - ; AF: 7DEE JNL L0 - ; B1: 4801F2 ADD RDX, RSI - ; B4: EBE9 JMP L0 - ; B6: L2: 0F0B0A BREAK 10 ; error trap - ; B9: 07 BYTE #X07 - ; BA: 08 BYTE #X08 ; DIVISION-BY-ZERO-ERROR - ; BB: FE9E03 BYTE #XFE, #X9E, #X03 ; RDI - ; BE: FE1E03 BYTE #XFE, #X1E, #X03 ; RSI - -Wait, it's actually *shorter*? What about all those extra variables the `zapf` -macro expands into? It turns out that SBCL is really quite good at optimizing -`let` statements and local variables. - -This is why I love Common Lisp. You can be rewriting the syntax of the language -and working on a tower of abstraction one moment, and looking at X86 assembly to -see what the hell the computer is actually *doing* the next. It's wonderful. - -[Getting Places]: http://malisper.me/2015/09/22/getting-places/ -[Zap]: http://malisper.me/2015/09/29/zap/ - - {% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2016/08/playing-with-syntax.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2016/08/playing-with-syntax.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,514 @@ ++++ +title = "Playing With Syntax" +snip = "Lisp lets you evolve your language." +date = 2016-08-19T13:15:00Z +draft = false + ++++ + +One of the things I love about Lisp is that it gives you the ability to change +and mold the syntax of the language to what you need. In this post I want to +look at the evolution of a little macro I've been playing around with for +a while now. + +Mutation is a fundamental concept in most programming languages. Functional +programming may be beautiful, but mutation is still useful (and fast). In most +languages assignment is done with an `=` or `:=` operator: + +```lisp +x = 10 +``` + +In Common Lisp the operator is named `setf` instead of `=` for historical +reasons, and of course it's used in prefix notation: + +```lisp +(setf x 10) +``` + +Aside from the prefix ordering, Common Lisp's syntax is already a bit more +elegant because you can set arbitrarily many variables without repeating the +assignment operator over and over again: + +```lisp +; Have to type out `=` three times +x = 10 +y = 20 +z = 30 + +; But only one `setf` required +(setf x 10 + y 20 + z 30) +``` + +`setf` assigns values one-by-one, in-order. Common Lisp also has `psetf` which +is "parallel `setf`". This evaluates all the values first, *then* sets all the +variables at once: + +```lisp +(let ((a 10) (b 20)) + (setf a 50 + b (+ 1 a)) ; => (50 51) + + (psetf a 90 + b (+ 1 a))) ; => (90 52) +``` + +Assignment is nice, but often you want to change the value of a variable by +taking its *current* value and computing something with it. If you're making +a ball that travels across the screen you might have something like this: + +```text +def move_ball(ball, distance): + ball.x = ball.x + distance + +(defun move-ball (ball distance) + (setf (slot-value ball 'x) + (+ (slot-value ball 'x) + distance))) +``` + +If you find `slot-value` too wordy to type, you could make a macro to get rid of +it (and save yourself a quote too). `.` is already used for something else so +you'd need to pick a different character: + +```lisp +(defmacro $ (instance slot-symbol) + `(slot-value ,instance ',slot-symbol)) + +(defun move-ball (ball distance) + (setf ($ ball x) + (+ ($ ball x) + distance))) +``` + +In practice, though, you rarely use `slot-value`. Instead you usually use +accessor functions: + +```lisp +(defun move-ball (ball distance) + (setf (ball-x ball) + (+ (ball-x ball) + distance))) +``` + +Anyway, back to assignment. What we wrote above *works*, but it's a bit +annoying to have to type the "place" that we're working with twice. To cut down +on this repetition many languages offer operators like `+=` and `++`: + +```text +x = x + 10 +⬇ +x += 10 + +y = y + 1 +⬇ +y++ +``` + +Common Lisp's version of these is called `incf`. + +```lisp +(incf x 10) +(incf y) +``` + +Notice how the s-expression syntax means we can use the same operator for both +of the forms, instead of needing separate `+=` and `++` operators. + +Other languages often have similar "in-place" operators for other numeric +operations like `-=`, `--`, `*=`, and so on. Common Lisp has `decf` for +subtraction, but doesn't have versions for multiplication or division built in. +But we can add them with a few macros: + +```lisp +(defmacro mulf (place factor) + `(setf ,place (* ,place ,factor))) + +(defmacro divf (place divisor) + `(setf ,place (/ ,place ,divisor))) +``` + +Unfortunately these are not quite right — we've committed the cardinal +macro-writing sin of splicing in the `place` expression multiple times. If +`place` has side effects they'll happen more times than expected. + +Luckily, Common Lisp has a macro that wraps up the boring process of handling +this for us. `define-modify-macro` lets us fix the problem: + +```lisp +(define-modify-macro mulf (factor) *) +(define-modify-macro divf (divisor) /) +``` + +Now we've got versions of `*=` and `/=` in Common Lisp. While we're here we +should do things *right* and allow `divf` to be used with just a place, which +will mean "set `place` to `1/place`": + +```lisp +(define-modify-macro divf (&optional divisor) + (lambda (value divisor) + (if (null divisor) + (/ 1 value) + (/ value divisor)))) + +(let ((x 10)) + (divf x 2) ; => 5 + (divf x) ; => 1/5 + (divf x)) ; => 5 +``` + +Note that we don't really need the `1` in `(/ 1 value)`, because Common Lisp's +`/` works the same way when you call it with a single argument. I'm not sure +what behavior would make sense for a unary `(mulf place)`, so let's ignore that +one and move on. + +So far we've been talking about the idea of setting a variable to the result of +computing some function on its current value. We used `define-modify-macro` to +define some support for extra functions, but defining a mutator macro for every +function we might possibly want to use could get tedious. What if we generalize +this a bit more? + +```lisp +(define-modify-macro callf (function) + (lambda (value function) + (funcall function value))) + +(let ((x -1)) + (callf x #'abs) ; => 1 + (callf x #'1+) ; => 2 + (callf x #'sin) ; => 0.9092974 + (callf x #'round) ; => 1 + (callf x #'-)) ; => -1 +``` + +Now we've got something a bit higher-level. With `callf` we can mutate +a variable with a unary function. But what about functions that need more +arguments? Again, Common Lisp does the right thing and handles `&rest` +parameters correctly in `define-modify-macro`: + +```lisp +(define-modify-macro callf (function &rest args) + (lambda (value function &rest args) + (apply function value args))) + +(let ((x -1)) + (callf x #'abs) ; => 1 + (callf x #'+ 10) ; => 11 + (callf x #'* 2 4)) ; => 88 + +(let ((foo (list 0 1 2 3))) + (callf foo #'reverse) + ; => (3 2 1 0) + + (callf foo #'append (list -1 -2 -3))) + ; => (3 2 1 0 -1 -2 -3) +``` + +That's better. Now we can mutate a variable by applying a function to its +current value and some other arguments. This might come in handy when we don't +want to bother defining a full-blown modification macro just to use it once or +twice. + +At this point we've got something similar to Michael Malis' [zap][] macro. +A minor difference is for `zap` the order of the function and the place are +reversed. I believe his reason was that it makes the resulting form look more +like the function call that is conceptually being made: + +```lisp +(zap #'+ x 5) +; ↓ ↓ ↓ +(setf x ( + x 5 )) +``` + +Unfortunately, because the place is no longer the first argument we can't use +`define-modify-macro` to handle the single-evaluation boilerplate for us any +more. Instead we need to use `get-setf-expander` and work with its results. +Malis' excellent [Getting Places][] article explains how that works, and if you +want to fully understand the rest of the code in this post you should read that +one first. + +The difference between `callf` and Malis' `zap` are just cosmetic. But they're +still not completely flexible. What if we want to call a function, but we need +the value to be an argument other than the first? Of course we could use +a `lambda`: + +```lisp +(let ((x (list :coffee :soda :tea))) + (callf x (lambda (l) (remove :soda l)))) + +``` + +But that is *really* ugly. It would be nice if we had a way to specify which +argument to the function should be the current value. This is Lisp, so we can +pick whatever syntax we want. What should we choose? Often a good first step +is to look at how other languages do things. Clojure's anonymous function +syntax is one possibility: + +```lisp +(map #(+ 10 %) [1 2 3]) +; => (11 12 13) +``` + +Clojure uses the magic symbol `%` as a placeholder for the argument. We can +modify Malis' `zap` macro to do the same. We'll call it `zapf`, and we'll swap +the place back into the first argument like all the other modify macros, since +the aesthetic reason no longer holds. The changes are in bold: + +
; Original version
+(defmacro zap (fn place &rest args)
+  (multiple-value-bind
+        (temps exprs stores store-expr access-expr)
+      (get-setf-expansion place)
+    `(let* (,@(mapcar #'list temps exprs)
+            (,(car stores)
+              (funcall ,fn ,access-expr ,@args)))
+       ,store-expr)))
+
+; New version
+(defmacro zapf (place fn &rest args)
+  (multiple-value-bind
+        (temps exprs stores store-expr access-expr)
+      (get-setf-expansion place)
+    `(let* (,@(mapcar #'list temps exprs)
+            (,(car stores)
+             (funcall ,fn ,@(substitute access-expr '% args))))
+      ,store-expr)))
+
+ +And now we can use `%` to say where the place's value should go: + +```lisp +(let ((x (list :coffee :soda :tea))) + (zapf x #'remove :soda %) + ; => (:coffee :tea) + + (zapf x #'cons :water %) + ; => (:water :coffee :tea) + + (zapf x #'append % (list :cocoa :chai))) + ; => (:water :coffee :tea :cocoa :chai) +``` + +Note that `substitute` will replace *all* occurrences of `%` with the access +expression, so we can do something like `(zapf x #'append % %)` if we want. + +Some people will find this new version unpleasant, because it effectively +captures the `%` variable. It's not hygienic, by design. I personally don't +mind it, and I like the brevity it allows. + +We're almost done, but I want to push things just a *bit* further. Let's +revisit the original example of the ball moving across the screen: + +```text +def move_ball(ball, distance): + ball.x = ball.x + distance + +(defun move-ball (ball distance) + (setf (ball-x ball) + (+ (ball-x ball) distance))) +``` + +Of course we can simply use `+=` or `incf` here: + +```text +def move_ball(ball, distance): + ball.x += distance + +(defun move-ball (ball distance) + (incf (ball-x ball) distance)) +``` + +Suppose we also wanted to make the ball wrap around the screen when it flies off +one side. We can do this using modulus: + +```text +def move_ball(ball, distance, screen_width): + ball.x += distance + ball.x %= screen_width + +(defun move-ball (ball distance screen-width) + (incf (ball-x ball) distance) + (callf (ball-x ball) #'mod % screen-width)) +``` + +We could define `modf` if we wanted to make that second call a bit nicer. But +let's take a moment to reflect. Notice how we're back to having to type out +`(ball-x ball)` twice again. It's better than typing it four times, but can we +do better? + +`zapf` currently takes a place, a function, and a list of arguments. What if we +made it more like `setf`, and just have it take a place and an expression? We +could still use `%` as a placeholder, but could let it appear anywhere in the +(possibly nested) expression form. + +One way to do this would be to simply replace `substitute` with `subst` in the +`zapf` code. (The difference between these extremely-confusingly-named +functions is that `substitute` replaces element of a sequence, and `subst` +replaces elements of a tree.) + +This, however, is the wrong strategy. Blindly replacing `%` everywhere in the +expression will work for many cases, but will break in edge cases like (god help +you) nested `zapf` forms. We could try to walk the code of the expression we +get, but down this path lies madness and implementation-specificity. + +The solution is much, much simpler. We can just use a normal `let` to capture +`%` in the expression: + +
; Final version
+(defmacro zapf (place expr)
+  (multiple-value-bind
+        (temps exprs stores store-expr access-expr)
+      (get-setf-expansion place)
+    `(let* (,@(mapcar #'list temps exprs)
+            (,(car stores)
+             (let ((% ,access-expr))
+               ,expr)))
+      ,store-expr)))
+
+ +And now `move-ball` only requires the bare minimum of typing: + +```lisp +(defun move-ball (ball distance screen-width) + (zapf (ball-x ball) + (mod (+ distance %) screen-width))) +``` + +For completeness it would be nice to allow `zapf` to take any number of +place/value pairs, just like `setf`. I'll leave this as an exercise for you to +try if you're interested. + +It took me a while to arrive at this form of the macro. I personally find it +lovely and readable. If you disagree, that's fine too! Lisp is a wonderful +substrate for building a language that fits how you think and talk about +problems. Play around with your own syntax and find out what feels most natural +for you. + +Before we leave, let's just ponder one more thing. Performance is often ignored +these days, but what if we care about making the computer go fast? The final +`zapf` macro is handy and expressive, but what cost have we paid for this +abstraction? + +First let's tell SBCL that we want to go fast and return to our original `setf` +strategy: + +```lisp +(deftype nonnegative-fixnum () + `(integer 0 ,most-positive-fixnum)) + +(deftype positive-fixnum () + `(integer 1 ,most-positive-fixnum)) + +(defstruct ball + (x 0 :type nonnegative-fixnum) + (y 0 :type nonnegative-fixnum)) + +(declaim (ftype (function (ball fixnum positive-fixnum)) + move-ball)) + +(defun move-ball (ball distance screen-width) + (declare (optimize (speed 3) (safety 0) (debug 0))) + (setf (ball-x ball) + (mod (+ distance (ball-x ball)) + screen-width))) +``` + +How does that turn out? + +```text +; disassembly for MOVE-BALL +; Size: 82 bytes. Origin: #x1005E5E12E +; 2E: 488B410D MOV RAX, [RCX+13] ; no-arg-parsing entry point +; 32: 48D1F8 SAR RAX, 1 +; 35: 498D3C00 LEA RDI, [R8+RAX] +; 39: 488BDE MOV RBX, RSI +; 3C: 48D1FB SAR RBX, 1 +; 3F: 4885DB TEST RBX, RBX +; 42: 7431 JEQ L3 +; 44: 488BC7 MOV RAX, RDI +; 47: 4899 CQO +; 49: 48F7FB IDIV RAX, RBX +; 4C: 4C8BC0 MOV R8, RAX +; 4F: 488BC2 MOV RAX, RDX +; 52: 48D1E0 SHL RAX, 1 +; 55: 4885C0 TEST RAX, RAX +; 58: 7510 JNE L2 +; 5A: L0: 488BD8 MOV RBX, RAX +; 5D: L1: 4889590D MOV [RCX+13], RBX +; 61: 488BD3 MOV RDX, RBX +; 64: 488BE5 MOV RSP, RBP +; 67: F8 CLC +; 68: 5D POP RBP +; 69: C3 RET +; 6A: L2: 4885FF TEST RDI, RDI +; 6D: 7DEB JNL L0 +; 6F: 488D1C30 LEA RBX, [RAX+RSI] +; 73: EBE8 JMP L1 +; 75: L3: 0F0B0A BREAK 10 ; error trap +; 78: 07 BYTE #X07 +; 79: 08 BYTE #X08 ; DIVISION-BY-ZERO-ERROR +; 7A: FE9E03 BYTE #XFE, #X9E, #X03 ; RDI +; 7D: FE9E01 BYTE #XFE, #X9E, #X01 ; RBX +``` + +Well that's not *too* bad. I'm not sure why SBCL still performs a check for +division by zero for the `mod` even though we said that `screen-width` can't +possibly be zero. But hey, we're at a manageable amount of assembly, which is +pretty nice for a high-level language! + +Now for the `zapf` version: + +```lisp +(defun move-ball (ball distance screen-width) + (declare (optimize (speed 3) (safety 0) (debug 0))) + (zapf (ball-x ball) + (mod (+ distance %) screen-width))) +``` + +Okay, let's bite the bullet. How bad are we talking? + +```text +; disassembly for MOVE-BALL +; Size: 70 bytes. Origin: #x1005F47C7B +; 7B: 488B410D MOV RAX, [RCX+13] ; no-arg-parsing entry point +; 7F: 48D1F8 SAR RAX, 1 +; 82: 488D3C03 LEA RDI, [RBX+RAX] +; 86: 4885F6 TEST RSI, RSI +; 89: 742B JEQ L2 +; 8B: 488BC7 MOV RAX, RDI +; 8E: 4899 CQO +; 90: 48F7FE IDIV RAX, RSI +; 93: 488BD8 MOV RBX, RAX +; 96: 488D0412 LEA RAX, [RDX+RDX] +; 9A: 4885C0 TEST RAX, RAX +; 9D: 750D JNE L1 +; 9F: L0: 48D1E2 SHL RDX, 1 +; A2: 4889510D MOV [RCX+13], RDX +; A6: 488BE5 MOV RSP, RBP +; A9: F8 CLC +; AA: 5D POP RBP +; AB: C3 RET +; AC: L1: 4885FF TEST RDI, RDI +; AF: 7DEE JNL L0 +; B1: 4801F2 ADD RDX, RSI +; B4: EBE9 JMP L0 +; B6: L2: 0F0B0A BREAK 10 ; error trap +; B9: 07 BYTE #X07 +; BA: 08 BYTE #X08 ; DIVISION-BY-ZERO-ERROR +; BB: FE9E03 BYTE #XFE, #X9E, #X03 ; RDI +; BE: FE1E03 BYTE #XFE, #X1E, #X03 ; RSI +``` + +Wait, it's actually *shorter*? What about all those extra variables the `zapf` +macro expands into? It turns out that SBCL is really quite good at optimizing +`let` statements and local variables. + +This is why I love Common Lisp. You can be rewriting the syntax of the language +and working on a tower of abstraction one moment, and looking at X86 assembly to +see what the hell the computer is actually *doing* the next. It's wonderful. + +[Getting Places]: http://malisper.me/2015/09/22/getting-places/ +[Zap]: http://malisper.me/2015/09/29/zap/ + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2016/09/iterate-averaging.html --- a/content/blog/2016/09/iterate-averaging.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,151 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "Customizing Common Lisp's Iterate: Averaging" - snip: "Don't loop, iterate!" - created: 2016-09-20 14:00:00 - %} - - {% block article %} - -When I first started learning Common Lisp, one of the things I learned was the -[loop macro][loop]. `loop` is powerful, but it's not extensible (at least not -*portably*) and [some people find it ugly][lol]. The [iterate][] library was -made to solve both of these problems. - -Unfortunately I haven't found many guides or resources on how to extend -`iterate`. The `iterate` manual describes the macros you need to use, but only -gives a few sparse examples. Sometimes it's helpful to see things in action. - -I've made a few handy extensions myself in the past couple of months, so -I figured I'd post about them in case someone else is looking for examples of -how to write their own `iterate` clauses and drivers. - -This first post will show how to make a `averaging` clause that keeps a running -average of a given expression during the loop. I've found it handy in a couple -of places. - -[loop]: http://www.gigamonkeys.com/book/loop-for-black-belts.html -[iterate]: https://common-lisp.net/project/iterate/ -[lol]: /media/images{{ parent_url }}/loop-macro.jpg - -[TOC] - -## End Result - -Before we look at the code, let's look at what we're aiming for: - - :::lisp - (iterate (for i :in (list 20 10 10 20)) - (averaging i)) - ; => - 15 - - (iterate (for l :in '((10 :foo) (20 :bar) (0 :baz))) - (averaging (car l) :into running-average) - (collect running-average)) - ; => - (10 15 10) - -Simple enough. The `averaging` clause takes an expression (and optionally -a variable name) and averages its value over each iteration of the loop. - -## Code - -There's not much code to `averaging`, but it does contain a few ideas that crop -up often when writing `iterate` extensions: - - :::lisp - (defmacro-clause (AVERAGING expr &optional INTO var) - "Maintain a running average of `expr` in `var`. - - If `var` is omitted the final average will be - returned instead. - - Examples: - - (iterate (for x :in '(0 10 0 10)) - (averaging x)) - => - 5 - - (iterate (for x :in '(1.0 1 2 3 4)) - (averaging (/ x 10) :into avg) - (collect avg)) - => - (0.1 0.1 0.13333334 0.17500001 0.22) - - " - (with-gensyms (count total) - (let ((average (or var iterate::*result-var*))) - `(progn - (for ,count :from 1) - (sum ,expr :into ,total) - (for ,average = (/ ,total ,count)))))) - -We use `defmacro-clause` to define the clause. Check [the iterate manual][man] -to learn more about the basics of that. - -The first thing to note is the big docstring, which describes how to use the -clause and gives some examples. I prefer to err on the side of providing *more* -information in documentation rather than less. People who don't need the -hand-holding can quickly skim over it, but if you omit information it can leave -people confused. [Your monitor isn't going to run out of ink][ink] and [you -type fast (right?)][typing] so be nice and just write the damn docs. - -Next up is selecting the name of the variable for the average. The `(or var -iterate::*result-var*)` pattern is one I use often when writing `iterate` -clauses. It's kind of weird that `*result-var*` isn't external in the `iterate` -package, but this idiom is explicitly mentioned in the manual so I suppose it's -fine to use. - -Finally, we *could* have written a simpler version of `averaging` that just -returned the result from the loop: - - :::lisp - (defmacro-clause (AVERAGING expr) - (with-gensyms (count total) - `(progn - (for ,count :from 1) - (sum ,expr :into ,total) - (finally (return (/ ,total ,count)))))) - -This would work, but doesn't let us see the running average during the course of -the loop. `iterate`'s built-in clauses like `collect` and `sum` usually allow -you to access the "in-progress" value, so it's good for our extensions to -support it too. - -[man]: https://common-lisp.net/project/iterate/doc/Rolling-Your-Own.html#Rolling-Your-Own -[ink]: http://www.bash.org/?105201 -[typing]: https://steve-yegge.blogspot.is/2008/09/programmings-dirtiest-little-secret.html - -## Debugging - -This clause is pretty simple, but more complicated ones can get a bit tricky. -When writing vanilla Lisp macros I usually end up writing the macro and then -`macroexpand-1`'ing a sample of it to make sure it's expanding to what I think -it should. - -As far as I can tell there's no simple way to macroexpand an `iterate` clause on -its own. This is really a pain in the ass when you're trying to debug them, so -I [hacked together][mxi] a `macroexpand-iterate` function for my own sanity. -It's not pretty, but it gets the job done: - - :::lisp - (macroexpand-iterate '(averaging (* 2 x))) - ; => - (PROGN - (FOR #:COUNT518 :FROM 1) - (SUM (* 2 X) :INTO #:TOTAL519) - (FOR ITERATE::*RESULT-VAR* = (/ #:TOTAL519 #:COUNT518))) - - (macroexpand-iterate '(averaging (* 2 x) :into foo)) - ; => - (PROGN - (FOR #:COUNT520 :FROM 1) - (SUM (* 2 X) :INTO #:TOTAL521) - (FOR FOO = (/ #:TOTAL521 #:COUNT520))) - -[mxi]: https://github.com/sjl/cl-losh/blob/55de0419a9b97a35943ce4884b598dbd99cc5670/losh.lisp#L1105-L1151 - - {% endblock article %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/2016/09/iterate-averaging.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/blog/2016/09/iterate-averaging.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,152 @@ ++++ +title = "Customizing Common Lisp's Iterate: Averaging" +snip = "Don't loop, iterate!" +date = 2016-09-20T13:45:00Z +draft = false + ++++ + +When I first started learning Common Lisp, one of the things I learned was the +[loop macro][loop]. `loop` is powerful, but it's not extensible and [some +people find it ugly][lol]. The [iterate][] library was made to solve both of +these problems. + +Unfortunately I haven't found many guides or resources on how to extend +`iterate`. The `iterate` manual describes the macros you need to use, but only +gives a few sparse examples. Sometimes it's helpful to see things in action. + +I've made a few handy extensions myself in the past couple of months, so +I figured I'd post about them in case someone else is looking for examples of +how to write their own `iterate` clauses and drivers. + +This first post will show how to make a `averaging` clause that keeps a running +average of a given expression during the loop. I've found it handy in a couple +of places. + +[loop]: http://www.gigamonkeys.com/book/loop-for-black-belts.html +[iterate]: https://common-lisp.net/project/iterate/ +[lol]: /media/images/blog/2016/09/loop-macro.jpg + +{{% toc %}} + +## End Result + +Before we look at the code, let's look at what we're aiming for: + +```lisp +(iterate (for i :in (list 20 10 10 20)) + (averaging i)) +; => +15 + +(iterate (for l :in '((10 :foo) (20 :bar) (0 :baz))) + (averaging (car l) :into running-average) + (collect running-average)) +; => +(10 15 10) +``` + +Simple enough. The `averaging` clause takes an expression (and optionally +a variable name) and averages its value over each iteration of the loop. + +## Code + +There's not much code to `averaging`, but it does contain a few ideas that crop +up often when writing `iterate` extensions: + +```lisp +(defmacro-clause (AVERAGING expr &optional INTO var) + "Maintain a running average of `expr` in `var`. + + If `var` is omitted the final average will be + returned instead. + + Examples: + + (iterate (for x :in '(0 10 0 10)) + (averaging x)) + => + 5 + + (iterate (for x :in '(1.0 1 2 3 4)) + (averaging (/ x 10) :into avg) + (collect avg)) + => + (0.1 0.1 0.13333334 0.17500001 0.22) + + " + (with-gensyms (count total) + (let ((average (or var iterate::*result-var*))) + `(progn + (for ,count :from 1) + (sum ,expr :into ,total) + (for ,average = (/ ,total ,count)))))) +``` + +We use `defmacro-clause` to define the clause. Check [the iterate manual][man] +to learn more about the basics of that. + +The first thing to note is the big docstring, which describes how to use the +clause and gives some examples. I prefer to err on the side of providing *more* +information in documentation rather than less. People who don't need the +hand-holding can quickly skim over it, but if you omit information it can leave +people confused. [Your monitor isn't going to run out of ink][ink] and [you +type fast (right?)][typing] so be nice and just write the damn docs. + +Next up is selecting the name of the variable for the average. The `(or var +iterate::*result-var*)` pattern is one I use often when writing `iterate` +clauses. It's kind of weird that `*result-var*` isn't external in the `iterate` +package, but this idiom is explicitly mentioned in the manual so I suppose it's +fine to use. + +Finally, we *could* have written a simpler version of `averaging` that just +returned the result from the loop: + +```lisp +(defmacro-clause (AVERAGING expr) + (with-gensyms (count total) + `(progn + (for ,count :from 1) + (sum ,expr :into ,total) + (finally (return (/ ,total ,count)))))) +``` + +This would work, but doesn't let us see the running average during the course of +the loop. `iterate`'s built-in clauses like `collect` and `sum` usually allow +you to access the "in-progress" value, so it's good for our extensions to +support it too. + +[man]: https://common-lisp.net/project/iterate/doc/Rolling-Your-Own.html#Rolling-Your-Own +[ink]: http://www.bash.org/?105201 +[typing]: https://steve-yegge.blogspot.is/2008/09/programmings-dirtiest-little-secret.html + +## Debugging + +This clause is pretty simple, but more complicated ones can get a bit tricky. +When writing vanilla Lisp macros I usually end up writing the macro and then +`macroexpand-1`'ing a sample of it to make sure it's expanding to what I think +it should. + +As far as I can tell there's no simple way to macroexpand an `iterate` clause on +its own. This is really a pain in the ass when you're trying to debug them, so +I [hacked together][mxi] a `macroexpand-iterate` function for my own sanity. +It's not pretty, but it gets the job done: + +```lisp +(macroexpand-iterate '(averaging (* 2 x))) +; => +(PROGN + (FOR #:COUNT518 :FROM 1) + (SUM (* 2 X) :INTO #:TOTAL519) + (FOR ITERATE::*RESULT-VAR* = (/ #:TOTAL519 #:COUNT518))) + +(macroexpand-iterate '(averaging (* 2 x) :into foo)) +; => +(PROGN + (FOR #:COUNT520 :FROM 1) + (SUM (* 2 X) :INTO #:TOTAL521) + (FOR FOO = (/ #:TOTAL521 #:COUNT520))) +``` + +[mxi]: https://github.com/sjl/cl-losh/blob/55de0419a9b97a35943ce4884b598dbd99cc5670/losh.lisp#L1105-L1151 + diff -r bbf39c61e3fe -r e7bc59b9ebda content/blog/index.html --- a/content/blog/index.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,6 +0,0 @@ -{% extends "skeleton/_listing.html" %} - -{% hyde - title: "Blog" - exclude: True -%} \ No newline at end of file diff -r bbf39c61e3fe -r e7bc59b9ebda content/feed.html --- a/content/feed.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1 +0,0 @@ -{% extends "skeleton/_atom.xml" %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/index.html --- a/content/index.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,5 +0,0 @@ -{% extends "_splash.html" %} - -{% hyde - title: "" -%} diff -r bbf39c61e3fe -r e7bc59b9ebda content/links.html --- a/content/links.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,39 +0,0 @@ - {% extends "_flatpage.html" %} - - {% hyde - title: "Links" - %} - - {% block article %} - -This page is a collection of links to blogs/resources I find interesting. - -It's mostly just a place for me to dump links that I want to remember to check -every now and then when I'm bored. I figured other people might find it -interesting too. - -Blogs ------ - -* [Habrador](http://blog.habrador.com/): Various game programming-related articles. -* [Martin O'Leary](http://mewo2.com/): Blog with some really good articles on procedural generation. -* [Michael Malis](http://malisper.me/): Lots of Lisp stuff (usually macro-related). -* [Paul Khuong](https://pvk.ca/): Mostly Lisp/SBCL-related stuff. -* [Peteris Krumins' Top 100 Books](http://www.catonmat.net/blog/top-100-books-part-one/): Great list of books to read. -* [Red Blob Games](http://www.redblobgames.com/): Wonderful blog with *really* good posts about lots of different video game-related topics. -* [The Digital Antiquarian](http://www.filfre.net/): Immense amount of information on the history of "computer entertainment". - -YouTube Channels ----------------- - -* [3Blue1Brown](https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw): Videos on various math topics. His series on linear algebra in particular is fantastic. -* [Coding Math](https://www.youtube.com/user/codingmath): A series of 50ish basic math lessons aimed at the kind of math you'll used in video games. Every lesson shows the concepts implemented in little Javascript demos, and it's *really* helpful to see how you actually *use* something once you learn the basic idea. -* [Makin' Stuff Look Good](https://www.youtube.com/channel/UCEklP9iLcpExB8vp_fWQseg): Videos about how to make things look good in Unity. The best part of the channel is the "shader case studies" where he looks at visual effects in various games and shows how to recreate them with a shader. - -Subreddits ----------- - -* [/r/proceduralgeneration](http://reddit.com/r/proceduralgeneration/) -* [/r/worldbuilding](http://reddit.com/r/worldbuilding/) - - {% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/links/index.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/links/index.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,36 @@ ++++ +date = "2016-06-20T13:17:43Z" +draft = false +title = "Links" + ++++ + +This page is a collection of links to blogs/resources I find interesting. + +It's mostly just a place for me to dump links that I want to remember to check +every now and then when I'm bored. I figured other people might find it +interesting too. + +Blogs +----- + +* [Habrador](http://blog.habrador.com/): Various game programming-related articles. +* [Martin O'Leary](http://mewo2.com/): Blog with some really good articles on procedural generation. +* [Michael Malis](http://malisper.me/): Lots of Lisp stuff (usually macro-related). +* [Paul Khuong](https://pvk.ca/): Mostly Lisp/SBCL-related stuff. +* [Peteris Krumins' Top 100 Books](http://www.catonmat.net/blog/top-100-books-part-one/): Great list of books to read. +* [Red Blob Games](http://www.redblobgames.com/): Wonderful blog with *really* good posts about lots of different video game-related topics. +* [The Digital Antiquarian](http://www.filfre.net/): Immense amount of information on the history of "computer entertainment". + +YouTube Channels +---------------- + +* [3Blue1Brown](https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw): Videos on various math topics. His series on linear algebra in particular is fantastic. +* [Coding Math](https://www.youtube.com/user/codingmath): A series of 50ish basic math lessons aimed at the kind of math you'll used in video games. Every lesson shows the concepts implemented in little Javascript demos, and it's *really* helpful to see how you actually *use* something once you learn the basic idea. +* [Makin' Stuff Look Good](https://www.youtube.com/channel/UCEklP9iLcpExB8vp_fWQseg): Videos about how to make things look good in Unity. The best part of the channel is the "shader case studies" where he looks at visual effects in various games and shows how to recreate them with a shader. + +Subreddits +---------- + +* [/r/proceduralgeneration](http://reddit.com/r/proceduralgeneration/) +* [/r/worldbuilding](http://reddit.com/r/worldbuilding/) diff -r bbf39c61e3fe -r e7bc59b9ebda content/projects/badwolf.html --- a/content/projects/badwolf.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,57 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "Bad Wolf" - snip: "A Vim color scheme." - created: 2012-02-12 15:06:01 - exclude: False - %} - - {% block article %} - -Bad Wolf is a Vim colorscheme pieced together from various colors I like. - -> I am the Bad Wolf. I create myself. -> I take the words. I scatter them in time and space. -> A message to lead myself here. - -You can [download][] the raw file, or use Pathogen with the [Mercurial -repository][hg] or the [Git repository][git]. - -[download]: http://bitbucket.org/sjl/badwolf/raw/tip/colors/badwolf.vim -[hg]: http://bitbucket.org/sjl/badwolf/ -[git]: http://github.com/sjl/badwolf/ - -Screenshots ------------ - -These screenshots may be out of date, but they'll at least give you a taste of -what you're in for. - - -### Python - -
- screenshot -
- -### HTML (Django Templates) - -
- screenshot -
- -### Clojure - -
- screenshot -
- -### Markdown - -
- screenshot -
- - - {% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/projects/clam.html --- a/content/projects/clam.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,25 +0,0 @@ -{% extends "_post.html" %} - -{% hyde -title: "Clam" -snip: "A Vim plugin for working with shell commands." -created: 2012-04-07 20:00:01 -exclude: False -%} - -{% block article %} - -[Clam][] is a small, lightweight Vim plugin for working with shell commands more -easily. - -It's open source (MIT/X11 licensed) on [BitBucket][] and [GitHub][]. - - - -[Clam]: http://vim-doc.heroku.com/view?https://raw.github.com/sjl/clam.vim/master/doc/clam.txt -[BitBucket]: http://bitbucket.org/sjl/clam.vim/ -[GitHub]: http://github.com/sjl/clam.vim/ - -{% endblock %} - - diff -r bbf39c61e3fe -r e7bc59b9ebda content/projects/clojure-postmark.html --- a/content/projects/clojure-postmark.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,23 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "clojure-postmark" - snip: "Use Postmark from Clojure." - created: 2011-10-10 00:00:03 - exclude: False - %} - - {% block article %} - -[clojure-postmark][] lets you talk to [Postmark][] from [Clojure][]. - -It's open source (MIT/X11 licensed) on [BitBucket][] and [GitHub][]. - -[clojure-postmark]: http://sjl.bitbucket.org/clojure-postmark/ -[Postmark]: http://postmarkapp.com/ -[Clojure]: http://clojure.org/ -[BitBucket]: http://bitbucket.org/sjl/d/ -[GitHub]: http://github.com/sjl/d/ - - {% endblock %} - diff -r bbf39c61e3fe -r e7bc59b9ebda content/projects/d.html --- a/content/projects/d.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,21 +0,0 @@ -{% extends "_post.html" %} - -{% hyde -title: "d" -snip: "Generate documentation and get on with your life." -created: 2012-01-23 20:00:01 -exclude: False -%} - -{% block article %} - -[d](http://sjl.bitbucket.org/d/) generates documentation from Markdown files -and lets you get on with your life. - -It's open source (MIT/X11 licensed) on [BitBucket][] and [GitHub][]. - -[BitBucket]: http://bitbucket.org/sjl/d/ -[GitHub]: http://github.com/sjl/d/ - -{% endblock %} - diff -r bbf39c61e3fe -r e7bc59b9ebda content/projects/django-hoptoad.html --- a/content/projects/django-hoptoad.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,35 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "django-hoptoad" - snip: "Now ponies can ride the toad too." - created: 2009-07-25 01:16:40 - exclude: True -%} - -{% block article %} - -django-hoptoad is some simple Middleware for letting [Django][]-driven -websites report their errors to [Hoptoad][]. Now [ponies][] can ride the toad -too. - -You can get it from [the repository on BitBucket][repo]. - -Check out the [documentation][docs] to find out how to use it. - -[Django]: {{links.django}} -[Hoptoad]: http://hoptoadapp.com/ -[ponies]: http://djangopony.com/ -[repo]: http://bitbucket.org/sjl/django-hoptoad/ -[docs]: http://sjl.bitbucket.org/django-hoptoad/ - -Suggestions ------------ - -This Middleware is a work in progress. If you have a suggestion or find a bug -please [add an issue][issues] or find me on [Twitter][twsl] and let me know. - -[issues]: http://bitbucket.org/sjl/django-hoptoad/issues/?status=new&status=open -[twsl]: {{links.twsl}} - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/projects/friendly-find.html --- a/content/projects/friendly-find.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,20 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "Friendly Find" - snip: "A friendlier find(1)." - created: 2012-11-19 19:00:01 - exclude: False - %} - -{% block article %} - -[Friendly Find][] is the friendly file finder. - -It's meant to be a more usable replacement for find(1). If you've used [ack][], -then `ffind` is to `find` as `ack` is to `grep`. - -[Friendly Find]: https://github.com/sjl/friendly-find -[ack]: http://betterthangrep.com/ - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/projects/fuego.html --- a/content/projects/fuego.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,39 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "Fuego" - snip: "Photographs of Alex. Studio." - created: 2009-01-19 01:25:20 - exclude: True -%} - -{% block article %} - -{% spaceless %} - - - -{% endspaceless %} - -Letting go. - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/projects/grabtweets.html --- a/content/projects/grabtweets.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,82 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "grabtweets" - snip: "A simple tool for backing up your tweets." - created: 2009-09-12 11:02:46 - exclude: True -%} - -{% block article %} - -`grabtweets` is a simple tool to backup your tweets. - -The code is in a [Mercurial repository][repository] on [BitBucket][]. - -[BitBucket]: http://bitbucket.org/ - -[TOC] - -Installing ----------- - -`grabtweets` requires [Python][] 2.6 or later. - -To install it, you can [download][] the latest version, or clone the Mercurial -repository: - - :::text - hg clone http://bitbucket.org/sjl/grabtweets/ - -You can put it anywhere you like. - -Using ------ - -`grabtweets` can back up your tweets and print tweets that it has already -archived. - -### Backing Up Tweets - -To back up your tweets, run `grabtweets` like this: - - :::console - $ grabtweets.py -u USERNAME FILE - -`grabtweets` will pull down the 200 newest tweets from USERNAME and store them -in FILE. - -You probably have more than 200 tweets. However, Twitter will only let you -pull down 200 at a time and hitting the server too fast will result in Twitter -cutting off your access for a while. - -To deal with this, it's best to set up `grabtweets` as a cron/launchd job that -runs every couple of hours. Each time it runs, it will pull down the newest -200 tweets, plus 200 older tweets if there are any available. - -It will take about `NUMBER_OF_TWEETS_YOU_HAVE/200` runs to finish archiving -your tweets. Just set it up and forget about it. - -### Printing Backed Up Tweets - -To print the tweets that `grabtweets` has already backed up, run it like this: - - :::console - $ grabtweets.py -p FILE - -[Python]: {{links.python}} -[download]: http://bitbucket.org/sjl/grabtweets/get/tip.zip - -Problems, Contributing, etc ---------------------------- - -`grabtweets` was hacked together in an hour. There are probably bugs. If you -find any, go ahead and create an issue in the [issue tracker][]. - -Want to fix something or add a feature? Great! Fork the [repository][] and -send me a pull request. - -[issue tracker]: http://bitbucket.org/sjl/grabtweets/issues/?status=new&status=open -[repository]: http://bitbucket.org/sjl/grabtweets/ - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/projects/gundo.html --- a/content/projects/gundo.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,15 +0,0 @@ -{% extends "_post.html" %} - -{% hyde -title: "Gundo" -snip: "Graph your Vim undo tree." -created: 2011-10-10 00:00:01 -exclude: False -%} - -{% block article %} - -[Gundo](http://sjl.bitbucket.org/gundo.vim/) is a Vim plugin that graphs your -undo tree. Check out the site for all the information. - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/projects/hg-paste.html --- a/content/projects/hg-paste.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,80 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "hg-paste" - snip: "Send diffs from Mercurial to various pastebin websites." - created: 2009-09-16 17:52:55 - exclude: True -%} - -{% block article %} - -hg-paste adds an `hg paste` command to Mercurial which can send diffs to -various pastebin websites for easy sharing. You can grab the code from the -[repository][] on BitBucket. - -It was inspired by [Pocoo][]'s [hgpaste][pocoopaste] extension, but is -designed to work with public pastebin websites instead of private -installations of [LodgeIt][]. - -[Pocoo]: http://www.pocoo.org/ -[pocoopaste]: http://dev.pocoo.org/hg/hgpaste/ -[LodgeIt]: http://dev.pocoo.org/projects/lodgeit/ - -[TOC] - -Installing ----------- - -Clone the repository: - - :::text - hg clone http://bitbucket.org/sjl/hg-paste/ - -Edit the `[extensions]` section in your `~/.hgrc` file: - - :::ini - [extensions] - paste = (path to)/hg-paste/paste.py - -Using the Command ------------------ - -To paste a diff of all uncommitted changes in the working directory: - - :::text - hg paste - -To paste the changes that revision `REV` made: - - :::text - hg paste -r REV - -To paste the changes between revisions `REV1` and `REV2`: - - :::text - hg paste -r REV1:REV2 - -If any files are specified only those files will be included in the diffs. - -Several options can be used to specify more metadata about the paste: - - :::text - hg paste --user Steve --title 'Progress on feature X' --keep - -The pastebin website to use can be specified with `--dest`. Currently only -[dpaste.com](http://dpaste.com/) and [dpaste.org](http://dpaste.org) are -supported. - -Questions, Comments, Suggestions --------------------------------- - -If you find any bugs, please create an issue in the [issue tracker][issues]. - -If you want to contribute (support for more pastebin websites would be great), -fork the [repository][] on BitBucket and send a pull request. - -[issues]: http://bitbucket.org/sjl/hg-paste/issues/?status=new&status=open -[repository]: http://bitbucket.org/sjl/hg-paste - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/projects/hg-prompt.html --- a/content/projects/hg-prompt.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,31 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "hg-prompt" - snip: "A Mercurial extension for adding repository info to your shell prompt." - created: 2009-06-19 22:25:17 -%} - -{% block article %} - -hg-prompt adds an 'hg prompt' command to Mercurial for viewing repository -information. It's designed to be used in a shell prompt. You can grab the code -from [the repository on BitBucket](http://bitbucket.org/sjl/hg-prompt/). - -Check out the [documentation][docs] to find out how to use it. - -Here's what it looks like: - -![My bash prompt while using hg-prompt.](/media/images/projects/{{ page.page_name }}/prompt.png "My bash prompt while using hg-prompt.") - -[docs]: http://sjl.bitbucket.org/hg-prompt/ - -Questions, Comments, Suggestions --------------------------------- - -The code was kind of thrown together in one night after I got tired of -chaining three or four hg runs together to get what I wanted. I'm sure it's -not perfect, so if you've got a way to improve it please [add an -issue](http://bitbucket.org/sjl/hg-prompt/issues/) and let me know. - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/projects/hgtab.html --- a/content/projects/hgtab.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,119 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "hgtab" - snip: "Smarter tab completion for Mercurial in bash." - created: 2009-03-10 20:22:44 - exclude: True -%} - -{% block article %} - -hgtab was created to provide smarter tab completion for Mercurial in bash. - -**UPDATE**: Apparently there's already [tab completion in -Mercurial](http://www.selenic.com/hg/file/11efa41037e2/contrib/bash_completion)! -It's way better than this, *use that*! It should probably be better documented -though – I posted this yesterday and it's already the top four results -for "mercurial tab completion" on Google... - -If I've missed something you'd really like tab completed, post an issue on the -issue tracker and I'll do my best to implement it. My bash scripting skills -are almost non-existent, so if you improve this (or port it to another shell) -feel free to send a pull request! - -You can download hgtab from its [BitBucket -repository](http://source.stevelosh.com/hgtab/). - -## Installing - -All you need to do to use hgtab is source it. There are three ways to do this, -ranging from easy and ugly to slightly less easy and more elegant. - -### OPTION 1: Easy, Ugly - -Copy the contents of `hgtab-bash.sh` into your `~/.bashrc` file. - -### OPTION 2: Harder, Less Ugly - -Download `hgtab-bash.sh` and place the following line somewhere in your -`~/.bashrc` file: - - :::bash - source ~/path/to/hgtab/hgtab-bash.sh - -### OPTION 3: Hardest, Elegant - -Clone this repository somewhere with this command: - - :::text - hg clone http://bitbucket.org/sjl/hgtab/ - -Add the following line to your `~/.bashrc` file: - - :::bash - source ~/path/to/hgtab/hgtab-bash.sh - -If you use this method you can update the file by pulling down the changes -from this repository: - - :::text - cd ~/path/to/hgtab - hg pull - -## Using - -Once hgtab is sourced it'll add some nifty tab completion features in your -shell. For example, you can tab complete Mercurial commands: - - :::console - sjl at grendel in test $ hg re - recover remove rename resolve revert - sjl at grendel in test $ hg re - -When you're using a command that wants a revision as a parameter, hitting tab -will complete on tags (and bookmarks) and branches. - - :::console - sjl at grendel in test $ hg branches - blue 20:f05ca6eb51a1 - red 16:8262f0345f41 (inactive) - default 15:67f959b8deb8 (inactive) - sjl at grendel in test $ hg book - * dev-1.0 20:f05ca6eb51a1 - sjl at grendel in test $ hg tags - tip 20:f05ca6eb51a1 - dev-1.0 20:f05ca6eb51a1 - beta 18:f3ab9bf730f7 - alpha 17:ca802019f04f - - sjl at grendel in test $ hg update - alpha beta blue default dev-1.0 red tip - sjl at grendel in test $ hg update b - beta blue - sjl at grendel in test $ hg update b - - sjl at grendel in test $ hg diff -r b - beta blue - sjl at grendel in test $ hg diff -r b - -It will also complete remote paths when you're pushing or pulling. - - :::console - sjl at grendel in test $ hg paths - alice-dev = http://alice-dev - alice-stable = http://alice-stable - bob = http://bob - default = ssh://hg@bitbucket.org/sjl/hgtag/ - - sjl at grendel in test $ hg push - alice-dev alice-stable bob default - sjl at grendel in test $ hg push - - sjl at grendel in test $ hg pull alice- - alice-dev alice-stable - sjl at grendel in test $ hg pull alice- - -Enjoy! - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/projects/index.html --- a/content/projects/index.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,6 +0,0 @@ -{% extends "skeleton/_listing.html" %} - -{% hyde - title: "Projects" - exclude: True -%} \ No newline at end of file diff -r bbf39c61e3fe -r e7bc59b9ebda content/projects/learnvimscriptthehardway.html --- a/content/projects/learnvimscriptthehardway.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,22 +0,0 @@ -{% extends "_post.html" %} - -{% hyde -title: "Learn Vimscript the Hard Way" -snip: "A short book about Vimscript." -created: 2011-10-10 00:32:00 -%} - -{% block article %} - -[Learn Vimscript the Hard Way][lvsthw] is a short book for users of the Vim -editor who want to learn how to customize Vim. - -It's currently a work in progress, so it's incomplete and and you should expect -mistakes. - -It will always be available for free online. Once it's finished you'll be able -to buy paperback and eBook versions for fairly cheap. - -[lvsthw]: http://learnvimscriptthehardway.stevelosh.com/ - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/projects/lindyjam-com.html --- a/content/projects/lindyjam-com.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,153 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "lindyjam.com" - snip: "A page for a local weekly swing dance event (and more)." - created: 2009-01-21 18:28:35 - exclude: True -%} - -{% block article %} - -A few days ago I released the new [Lindy Jam][] website. Lindy Jam is a weekly -swing dance in Rochester, NY. I wanted to create a site that was useful for -those that attend the event but also more than just a "here's when it is and -how to get there" page. - -[Lindy Jam]: http://lindyjam.com/ - -![lindyjam.com screenshot](/media/images/projects/{{ page.page_name }}/lindyjamcom-splash.png "Screenshot of the Lindy Jam site.") - -Design ------- - -Let me preface this section by saying: "I'm not a graphic designer." Yes, I -appreciate good design and typography, but I simply haven't studied it as much -as I'd like. That might change in the future, but for now I'll make due with -the knowledge I have. - -There are very few images used on the site; the background stripes and the -splash page photo(s) comprise all of them. I did this for a few reasons: - -* It saved me time. -* The site loads faster. -* The site scales to larger or smaller sizes very gracefully. - -I tried to keep things as clean as possible while still being useful. Comments -and error messages are displayed immediately, without reloading the page. This -makes it easier to see what you've done (or what went wrong) right off the -bat. - -I'm not 100% pleased with how it came out, but it will do until I learn more -about design. If you have advice or suggestions about the design, please -[email me][]! - -[email me]: mailto:steve@stevelosh.com - -Pages ------ - -### About - -The most important part of the site is probably the [About][] page, because it -tells people when and where the event happens. It's as simple and to the point -as I could make it so people can get the information they're looking for right -away. It's linked right from the splash page so it's easier to find. - -[About]: http://lindyjam.com/about/ - -### DJ Schedule - -Another important piece of the site is the [DJ Schedule][] page. Every week -someone can sign up to DJ the event. They get in for free, and we get to hear -a wider variety of music. The schedule page makes it easy to sign up for a -week in the near future. - -When you sign up to DJ you can also enter your email address or cell phone -number (or both) if you'd like to be reminded. I know that sometimes I've -almost forgotten to bring my laptop (and arrive right on time), so I hope that -this feature will be useful to other DJs too. - -[DJ Schedule]: http://lindyjam.com/schedule/ - -### Blog - -One thing I believe our Lindy Hop scene lacks is a place for people to write -thoughtful, well-crafted articles about the scene. I hope the Lindy Jam -[Blog][] can become such a place. - -It's a fairly simple blog; entries and comments are written in [Markdown][] so -authors don't need to know HTML to make their articles look nice. Anyone can -comment without signing up for anything, so I hope people will take advantage -of that and discuss the articles. - -[Blog]: http://lindyjam.com/blog/ -[Markdown]: {{links.markdown}} - -### Links - -Most sites like this have a list of links to other sites that their viewers -might be interested in, and this site is no different. The [Links][] page is a -simple, clean list of links that anyone can contribute to. If someone knows of -a great page we haven't listed, they can submit it and it will appear on the -page as soon as it's been approved. - -[Links]: http://lindyjam.com/links/ - -### Quotes - -Along the top of every page is a random quote. Right now there aren't very -many, but I hope that people will submit some more by going to the [Quotes][] -page or by text messaging them to . - -I think the quotes add a more personal touch and allowing anyone to add one -can really make it a "community" site. - -[Quotes]: http://lindyjam.com/quotes/ - -### RSS - -We nerds love our [RSS][] feeds, so I've implemented a couple of different -feeds that people can use to stay up to date. - -[RSS]: http://lindyjam.com/rss/ - -Implementation --------------- - -The site was built with [Django][] because it makes it very, very easy to get -a dynamic site like this up and running quickly. I use [Fabric][] to deploy it -(check out [this -entry](http://stevelosh.com/blog/entry/2009/1/15/deploying-site-fabric-and-mercurial/) -on my personal blog for more information about that). All of the fancy effects -use [jQuery][] so I can avoid programming them from scratch in javascript (and -tearing my hair out). - -One thing I would do differently if I were to design the site all over again -is use a CSS framework like [Blueprint][]. I used Blueprint for my personal -site and it made things far, far easier than writing the layout in CSS from -scratch. I definitely recommend it if you want to keep your sanity. - -The site itself is hosted on [WebFaction][]. I use them to host my own site -too and they're fantastic. Setting up a Django or Rails (or any other type) of -site is quick and easy, and they're very cheap. If you're looking for a web -host you should check them out. - -The site is open source if you'd like to take a look at what makes it tick. -You can browse or download the code from the [Mercurial repository][] on -BitBucket. - -[Django]: {{links.django}} -[Fabric]: {{links.fabric}} -[jQuery]: http://jquery.com/ -[Blueprint]: http://www.blueprintcss.org/ -[WebFaction]: {{links.webfaction}} -[Mercurial repository]: http://bitbucket.org/sjl/lindyjam/ - -Feedback --------- - -If you have any comments or questions about the site please let me know! You -can post them here or [email me][]. - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/projects/peat.html --- a/content/projects/peat.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,16 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "Peat" - snip: "Peat repeats commands." - created: 2012-11-19 19:00:02 - exclude: False - %} - -{% block article %} - -[Peat][] is a small utility for easily repeating a command when files change. - -[Peat]: https://github.com/sjl/peat - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/projects/roul.html --- a/content/projects/roul.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,22 +0,0 @@ -{% extends "_post.html" %} - -{% hyde -title: "Roul" -snip: "A tiny Clojure library for working with random numbers." -created: 2012-04-08 15:45:01 -exclude: False -%} - -{% block article %} - -[Roul][] is a little Clojure library that makes working with random numbers in -Clojure less painful. - -It's open source (MIT/X11 licensed) on [BitBucket][] and [GitHub][]. - -[Roul]: http://sjl.bitbucket.org/roul/ -[BitBucket]: http://bitbucket.org/sjl/roul/ -[GitHub]: http://github.com/sjl/roul/ - -{% endblock %} - diff -r bbf39c61e3fe -r e7bc59b9ebda content/projects/splice.html --- a/content/projects/splice.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,16 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "Splice" - snip: "A Vim plugin for resolving three-way merge conflicts." - created: 2011-10-10 00:00:02 - exclude: False - %} - - {% block article %} - -[Splice](http://sjl.bitbucket.org/splice.vim/) is a Vim plugin that helps -you resolve three-way merge conflicts. Check out the site for more information. - - {% endblock %} - diff -r bbf39c61e3fe -r e7bc59b9ebda content/projects/stevelosh-com.html --- a/content/projects/stevelosh-com.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,102 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "stevelosh.com" - snip: "This website." - created: 2009-01-11 15:40:26 - exclude: True -%} - -{% block article %} - -For a long time I used [Squarespace][] to create and host my personal website. -It's a great service, and I highly recommend it to anyone that needs a simple, -easy, beautiful site. - -Recently, however, I've started learning more and more about web design and I -decided to completely revamp my website and build it myself from the ground -up. This is the result. - -Design ------- - -I like simplicity. In all areas of my life (music, dancing, photography, -programming, design, etc) I'm striving to make everything I do as simple, -clean, and elegant as possible. That's why I went with such a minimal design -for the site. - -I haven't used any graphics in the layout for a few reasons. First, I'm not -very good at making them. Second, I want the site to load quickly, even on -phones, and scale gracefully for accessibility. Third, I wanted the interface -to be as uncluttered as possible so the content of the site would be front and -center. - -I also stuck to a minimum of color. The only items that have any color on the -site are any photographs that I post and the links. I did this because I'm not -a graphic designer and so I haven't studied color enough to know how to use it -well. - -One last thing I tried to keep in mind was sticking to a baseline rhythm for -some elements of the site. The home page, blog entry list, and project list -all line up with the links in the right sidebar. I think it gives the layout a -more stable feel. - -Implementation ------------- - -This website was written in [Python][] using the [Django][] framework. Django -makes it really easy to write useful web apps quickly and cleanly. I love it. - -[Blueprint][] kept me sane while writing the CSS. Grid layouts are no longer a -nightmare! - -I rely pretty heavily on [Markdown][] throughout the site. The blog entries -and static pages are all written in Markdown and parsed to XHTML later. This -might not be the fastest way to do things but processor time is cheap and my -time isn't. - -The Markdown-based editor and preview for commenting is the wonderful [WMD][]. -It's clean, simple, and does exactly what I want it to do. Most of the other -Javascript magic comes from [jQuery][]. I use the [jQuery form validation -plugin][form validation] in a couple of places to make checking input easier. - -For site statistics I'm relying on the fantastic [Mint][]. It's got a gorgeous -interface and is absurdly simple to install. Plus it's hosted on my own host -so I have control. It's also just a one time fee, unlike some other site stat -packages that charge monthly. - -My development process is pretty simple. I use [TextMate][] to edit the code -and a [Mercurial][] repository to store it and protect against accidents. To -deploy, I push my local changes to a private repository on [Bitbucket.org][], -and then use [Fabric][] to pull those changes down to the server and restart -if necessary. - -The site is hosted by [WebFaction][]. If you're looking for a web host for -Django or Rails sites (or any other kind, really) you should definitely -consider them. Not only do they make installing Django or Rails very easy, -they're absurdly cheap: 10gb disk space and 600gb/month bandwidth is less than -$10 a month. I'd recommend them to anyone. - -It's Open Source, Too! ----------------------- - -Want to see what makes this site tick? Want to make it tick more smoothly? -Feel free to head over to and take a look -at the code. - -[Squarespace]: http://www.squarespace.com/ -[Python]: http://python.org -[Django]: {{links.django}} -[Blueprint]: http://www.blueprintcss.org/ -[Markdown]: {{links.markdown}} -[WMD]: http://wmd-editor.com/ -[jQuery]: http://jquery.com -[form validation]: http://bassistance.de/jquery-plugins/jquery-plugin-validation/ -[TextMate]: http://macromates.com/ -[Mercurial]: {{links.mercurial}} -[Bitbucket.org]: http://www.bitbucket.org/ -[Fabric]: {{links.fabric}} -[Mint]: http://haveamint.com/ -[WebFaction]: {{links.webfaction}} - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/projects/t.html --- a/content/projects/t.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,257 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "t" - snip: "A command-line todo list manager for people that want to finish tasks, not organize them." - created: 2009-09-11 19:03:29 -%} - -{% block article %} - -`t` is a command-line todo list manager for people that want to *finish* -tasks, not organize them. - -The code is in a [Mercurial repository][] on [BitBucket][]. - -[BitBucket]: http://bitbucket.org/ - - -![Screenshot of t](/media/images/projects/t/t-screenshot.png "Screenshot of t") - -[TOC] - -Why t? ------- - -Yeah, I know, *another* command-line todo list manager. Several others already -exist ([todo.txt][] and [TaskWarrior][] come to mind), so why make another -one? - -[todo.txt]: http://ginatrapani.github.com/todo.txt-cli/ -[TaskWarrior]: http://taskwarrior.org/projects/show/taskwarrior/ - -### It Does the Simplest Thing That Could Possibly Work - -Todo.txt and TaskWarrior are feature-packed. They let you tag tasks, split -them into projects, set priorities, order them, color-code them, and much -more. - -**That's the problem.** - -It's easy to say "I'll just organize my todo list a bit" and spend 15 minutes -tagging your tasks. In those 15 minutes you probably could have *finished* a -couple of them. - -`t` was inspired by [j][]. It's simple, messy, has almost no features, and is -extremely effective at the one thing it does. With `t` the only way to make -your todo list prettier is to **finish some damn tasks**. - -[j]: http://github.com/rupa/j2/ - -### It's Flexible - -`t`'s simplicity makes it extremely flexible. - -Want to edit a bunch of tasks at once? Open the list in a text editor. - -Want to view the lists on a computer that doesn't have `t` installed? Open the -list in a text editor. - -Want to synchronize the list across a couple of computers? Keep your task -lists in a [Dropbox][] folder. - -Want to use it as a distributed bug tracking system like [BugsEverywhere][]? -Make the task list a `bugs` file in the project repository. - -[Dropbox]: https://www.getdropbox.com/ -[BugsEverywhere]: http://bugseverywhere.org/ - -### It Plays Nice with Version Control - -Other systems keep your tasks in a plain text file. This is a good thing, and -`t` follows their lead. - -However, some of them append new tasks to the end of the file when you create -them. This is not good if you're using a version control system to let more -than one person edit a todo list. If two people add a task and then try to -merge, they'll get a conflict and have to resolve it manually. - -`t` uses random IDs (actually SHA1 hashes) to order the todo list files. Once -the list has a couple of tasks in it, adding more is far less likely to cause -a merge conflict because the list is sorted. - -Installing t ------------- - -`t` requires [Python][] 2.5 or newer, and some form of UNIX-like shell (bash -works well). It works on Linux, OS X, and Windows (with [Cygwin][]). - -[Python]: {{links.python}} -[Cygwin]: http://www.cygwin.com/ - -Installing and setting up `t` will take about one minute. - -First, [download][] the newest version or clone the Mercurial repository ( `hg -clone http://bitbucket.org/sjl/t/` ). Put it anywhere you like. - -[download]: http://bitbucket.org/sjl/t/get/tip.zip - -Next, decide where you want to keep your todo lists. I put mine in `~/tasks`. -Create that directory: - - :::text - mkdir ~/tasks - -Finally, set up an alias to run `t`. Put something like this in your -`~/.bashrc` file: - - :::bash - alias t='python ~/path/to/t.py --task-dir ~/tasks --list tasks' - -Make sure you run `source ~/.bashrc` or restart your terminal window to make -the alias take effect. - -Using t -------- - -`t` is quick and easy to use. - -### Add a Task - -To add a task, use `t [task description]`: - - :::console - $ t Clean the apartment. - $ t Write chapter 10 of the novel. - $ t Buy more beer. - $ - -### List Your Tasks - -Listing your tasks is even easier -- just use `t`: - - :::console - $ t - 9 - Buy more beer. - 30 - Clean the apartment. - 31 - Write chapter 10 of the novel. - $ - -`t` will list all of your unfinished tasks and their IDs. - -### Finish a Task - -After you're done with something, use `t -f ID` to finish it: - - :::console - $ t -f 31 - $ t - 9 - Buy more beer. - 30 - Clean the apartment. - $ - -### Edit a Task - -Sometimes you might want to change the wording of a task. You can use `t -e ID -[new description]` to do that: - - :::console - $ t -e 30 Clean the entire apartment. - $ t - 9 - Buy more beer. - 30 - Clean the entire apartment. - $ - -Yes, nerds, you can use sed-style substitution strings: - - :::console - $ t -e 9 /more/a lot more/ - $ t - 9 - Buy a lot more beer. - 30 - Clean the entire apartment. - $ - -### Delete the Task List if it's Empty - -If you keep your task list in a visible place (like your desktop) you might -want it to be deleted if there are no tasks in it. To do this automatically -you can use the `--delete-if-empty` option in your alias: - - :::bash - alias t='python ~/path/to/t.py --task-dir ~/Desktop --list todo.txt --delete-if-empty' - -Tips and Tricks ---------------- - -`t` might be simple, but it can do a lot of interesting things. - -### Count Your Tasks - -Counting your tasks is simple using the `wc` program: - - :::console - $ t | wc -l - 2 - $ - -### Put Your Task Count in Your Bash Prompt - -Want a count of your tasks right in your prompt? Edit your `~/.bashrc` file: - - :::bash - export PS1="[$(t | wc -l | sed -e's/ *//')] $PS1" - -Now you've got a prompt that looks something like this: - - :::console - [2] $ t -f 30 - [1] $ t Feed the cat. - [2] $ - -### Multiple Lists - -`t` is for people that want to *do* tasks, not organize them. With that said, -sometimes it's useful to be able to have at least *one* level of organization. -To split up your tasks into different lists you can add a few more aliases: - - :::bash - alias g='python ~/path/to/t.py --task-dir ~/tasks --list groceries' - alias m='python ~/path/to/t.py --task-dir ~/tasks --list music-to-buy' - alias w='python ~/path/to/t.py --task-dir ~/tasks --list wines-to-try' - -### Distributed Bugtracking - -Like the idea of distributed bug trackers like [BugsEverywhere][], but don't -want to use such a heavyweight system? You can use `t` instead. - -Add another alias to your `~/.bashrc` file: - - :::bash - alias b='python ~/path/to/t.py --task-dir . --list bugs' - -Now when you're in your project directory you can use `b` to manage the list -of bugs/tasks for that project. Add the `bugs` file to version control and -you're all set. - -Even people without `t` installed can view the bug list, because it's plain -text. - -Problems, Contributions, Etc ----------------------------- - -`t` was hacked together in a couple of nights to fit my needs. If you use it -and find a bug, please let me know. - -If you want to request a feature feel free, but remember that `t` is meant to -be simple. If you need anything beyond the basics you might want to look at -[todo.txt][] or [TaskWarrior][] instead. They're great tools with lots of -bells and whistles. - -If you want to contribute code to `t`, that's great! Fork the [Mercurial -repository][] on BitBucket or the [git mirror][] on GitHub and send me a pull -request. - -[Mercurial repository]: http://bitbucket.org/sjl/t/ -[git mirror]: http://github.com/sjl/t/ - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/projects/threesome.html --- a/content/projects/threesome.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,15 +0,0 @@ - {% extends "_post.html" %} - - {% hyde - title: "Threesome" - snip: "A Vim plugin for resolving three-way merge conflicts." - created: 2011-10-10 00:00:02 - exclude: True - %} - - {% block article %} - -Threesome is now called [Splice](/projects/splice/). - - {% endblock %} - diff -r bbf39c61e3fe -r e7bc59b9ebda content/projects/typkov.html --- a/content/projects/typkov.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,28 +0,0 @@ -{% extends "_post.html" %} - -{% hyde -title: "Typkov" -snip: "Create typing lessons from your own writing." -created: 2012-01-23 00:00:01 -exclude: False -%} - -{% block article %} - -[Typkov](http://typkov.stevelosh.com) is a simple webapp that generates typing -lessons from your own writing. - -Instead of practicing random words from the dictionary you can practice with the -kind of stuff you actually write in real life. It works surprisingly well. - -It's open source (MIT/X11 licensed) on [BitBucket][] and [GitHub][]. - -I also [recorded myself][rec] as I made the initial version, so if you like -watching programmers work you might like that (though I'm pretty new to -Clojure/Noir). - -[BitBucket]: http://bitbucket.org/sjl/typkov/ -[GitHub]: http://github.com/sjl/typkov/ -[rec]: http://youtu.be/uTwnoZEw7FE - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/projects/vitality.html --- a/content/projects/vitality.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,22 +0,0 @@ -{% extends "_post.html" %} - -{% hyde -title: "Vitality" -snip: "Make Vim play nicely with iTerm 2 and tmux." -created: 2012-04-12 20:00:01 -exclude: False -%} - -{% block article %} - -[Vitality][] is a Vim plugin that makes Vim play more nicely with iTerm 2 and tmux. - -It's open source (MIT/X11 licensed) on [BitBucket][] and [GitHub][]. - -[Vitality]: http://vim-doc.heroku.com/view?https://raw.github.com/sjl/vitality.vim/master/doc/vitality.txt -[BitBucket]: http://bitbucket.org/sjl/vitality.vim/ -[GitHub]: http://github.com/sjl/vitality.vim/ - -{% endblock %} - - diff -r bbf39c61e3fe -r e7bc59b9ebda content/projects/women-in-water.html --- a/content/projects/women-in-water.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,47 +0,0 @@ -{% extends "_post.html" %} - -{% hyde - title: "Women in Water" - snip: "Photographs of women in Lake Ontario." - created: 2009-08-04 18:57:36 - exclude: True -%} - -{% block article %} - - - -This is still a work in progress. - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/resume.html --- a/content/resume.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,220 +0,0 @@ -{% extends "_flatpage.html" %} - -{% hyde - title: "Résumé" - exclude: True -%} - -{% block article %} - -I'm Steve. I'm a programmer from Rochester, NY. I graduated from [Rochester -Institute of Technology][rit] in 2008 with a Bachelor's degree in [Computer -Science][ritcs]. - -I'm currently most interested in rapid web development, version control, and -artificial intelligence. I'm not looking for full time work at the moment but -I'm available for interesting freelance projects. - -If you'd *really* like to get to know me you should look at the [projects][] -and [blog posts][] I've written, or cut to the chase and look at my [code][]. - -[rit]: http://rit.edu/ -[ritcs]: http://www.cs.rit.edu/ -[projects]: /projects/ -[blog posts]: /blog/ -[code]: http://bitbucket.org/sjl/ - -## Skills & Interests - -I'm passionate about programming. It may sound odd, but there are aspects of -programming that I find beautiful (in every sense of the word). - -### Languages - -My favorite programming language at the moment is [Python][]. It's elegant, -readable, powerful, and makes it easy to *get things done*. - -I've also had experience with [Java][], [C][], [C++][], [Lisp][], [SQL][] and -[PL/SQL][], [JavaScript][], [Groovy][], and [bash scripting][]. I've used a -number of markup and templating languages, including [XHTML][], [XML][], -[CSS][], [JSON][], [Markdown][], [Django][]'s template system, and [Jinja2][]. - -In my free time I'm working on learning [R][]. - -[Python]: http://www.python.org/ -[Java]: http://java.sun.com/ -[C]: http://en.wikipedia.org/wiki/C_(programming_language) -[C++]: http://en.wikipedia.org/wiki/C%2B%2B -[Lisp]: http://en.wikipedia.org/wiki/Lisp_(programming_language) -[SQL]: http://en.wikipedia.org/wiki/SQL -[PL/SQL]: http://en.wikipedia.org/wiki/PL/SQL -[JavaScript]: http://en.wikipedia.org/wiki/JavaScript -[Groovy]: http://groovy.codehaus.org/ -[bash scripting]: http://www.gnu.org/software/bash/ -[XHTML]: http://www.w3.org/TR/xhtml1/ -[XML]: http://www.w3.org/XML/ -[CSS]: http://www.w3.org/TR/CSS/ -[JSON]: http://www.json.org/ -[Markdown]: {{links.markdown}} -[Jinja2]: http://jinja.pocoo.org/2/ -[R]: http://www.r-project.org/ - -### Web Development - -For web development I gravitate toward the [Django][] framework. Django bills -itself as "The Web framework for professionals with deadlines" and this -strikes a chord with me. - -I've used the [CherryPy][] framework for smaller (but still dynamic) projects -that don't need the additional functionality of Django. - -I hate writing repetitive, difficult-to-maintain code. Even when writing -completely static websites I prefer to take advantage of templating languages. -To do that I use [Hyde][] and [Blatter][], both of which generate static HTML -from templates. - -I've used several CSS frameworks including [Blueprint][], [Tripoli][] and -[aardvark legs][]. Of these I now prefer aardvark legs because it makes -setting up a beautiful vertical rhythm simple and doesn't impose itself on the -horizontal layout. - -When writing [JavaScript][] I use [jQuery][] to eliminate a lot of the -tediousness and create elegant code. - -[Django]: {{links.django}} -[CherryPy]: {{links.cherrypy}} -[Blueprint]: http://www.blueprintcss.org/ -[Tripoli]: http://devkick.com/lab/tripoli/ -[aardvark legs]: {{links.aardvarklegs}} -[Hyde]: http://github.com/lakshmivyas/hyde -[Blatter]: {{links.blatter}} -[jQuery]: http://jquery.com/ - -### Version Control - -I think version control is essential for a project of almost any size, and -lately the concepts behind it have captured my interest. - -In my personal work I use [Mercurial][] (usually with [BitBucket][]). I've -[contributed][] several minor patches to the Mercurial core and written two -extensions: [hg-prompt][] and [hg-paste][]. - -I know my way around [Subversion][], [CVS][], and [git][] as well. - -[Mercurial]: {{links.mercurial}} -[BitBucket]: http://bitbucket.org/ -[Subversion]: http://subversion.tigris.org/ -[CVS]: http://www.nongnu.org/cvs/ -[git]: http://git-scm.com/ -[contributed]: http://selenic.com/repo/hg/log?rev=Steve+Losh -[hg-prompt]: /projects/hg-prompt/ -[hg-paste]: /projects/hg-paste/ - -### How I Work - -For my personal work I use [Mac OS X][]. I'm comfortable in [Linux][] and can -deal with [Windows][] (though I don't enjoy it). - -I use [TextMate][] and [vim][] for coding, [CSSEdit][] when designing, -[Mercurial][] for version control, [t][] to manage my personal tasks, and -deploy through [SSH][] with [fabric][]. - -[Mac OS X]: http://www.apple.com/macosx/ -[TextMate]: http://macromates.com/ -[SSH]: http://en.wikipedia.org/wiki/Secure_Shell -[fabric]: {{links.fabric}} -[Linux]: http://www.linux.org/ -[Windows]: http://www.microsoft.com/WINDOWS/ -[vim]: http://www.vim.org/ -[CSSEdit]: http://macrabbit.com/cssedit/ -[t]: /projects/t/ - -## Personal Projects - -You can find some of my pet projects on the [projects][] page. - -Lately I've been working on a few new things that aren't quite ready for -general use, but are getting close: - -* [hg-review][]: a code-review extension for Mercurial. -* [LindyHub][]: a site like BitBucket or GitHub for dancers. -* [gorilla][]: a packaging system for Python. -* [tinpan][]: a language-agnostic, vcs-agnostic continuous integration system. - -[hg-review]: http://bitbucket.org/sjl/hg-review/ -[LindyHub]: http://test.lindyhub.com/ -[gorilla]: http://bitbucket.org/sjl/gorilla/ -[tinpan]: http://bitbucket.org/sjl/tinpan/ - -## Full-Time Work Experience - -I completed two six-month [co-ops][] while at [RIT][]. Since I graduated I've -worked full time for about a year and a half. - -[co-ops]: https://www.rit.edu/about/coop_careers.html -[RIT]: http://rit.edu/ - -### Senior Software Engineer at [Dumbwaiter Design][DWD] - -*Henrietta, NY since January 2010.* - -At [Dumbwaiter][DWD] I work with our amazing designers to create beautiful websites backed by [Django][] and [Python][]. - -[DWD]: http://dwaiter.com/ - -### Software Engineer at [PAETEC][] - -*Fairport, NY from June 2008 to January 2010.* - -At [PAETEC][] I maintained and implemented new features for web applications -in [Java][] using the [Oracle E-Business Suite][oebs] and worked on underlying -[SQL][] and [PL/SQL][] code for our databases. - -[PAETEC]: http://paetec.com/ -[oebs]: http://www.oracle.com/applications/e-business-suite.html - -### Data Architecture Co-op at [Excellus BlueCross BlueShield][] - -*Rochester, NY from June 2007 to November 2007.* - -At [Excellus][] I developed software in [Java][] to interact with and manage -databases of customer and provider information stored on a mainframe. I also -created and updated [JUnit][] and [Jemmy][] tests for this software. - -[Excellus BlueCross BlueShield]: https://www.excellusbcbs.com/ -[Excellus]: https://www.excellusbcbs.com/ -[JUnit]: http://www.junit.org/ -[Jemmy]: http://jemmy.dev.java.net/ - -### Managed Services Programmer at [RightNow Technologies][] - -*Pittsford, NY from June 2006 to November 2006.* - -At [RightNow][] I developed a set of tools to test the effectiveness of -voice-automated telephone systems. These tools were written in a combination -of [bash scripting][] and [Python][]. I assisted in generating statistics -about the effectiveness of these systems for customers. - -[RightNow Technologies]: http://www.rightnow.com/ -[RightNow]: http://www.rightnow.com/ - -## Contact Me - -If you'd like to get in touch with me, you can use any method you like. - -My **email address** is: -My **phone number** is: (570) 417-1392 -My **skype username** is: steve.losh -My **mailing address** is: 150 Park Avenue, Apartment 7, Rochester, NY 14607 - -You can also find me on [Twitter][], [Flickr][], [BitBucket][], [GitHub][], -[BrightKite][], or [DjangoPeople][]. - -[Twitter]: http://twitter.com/stevelosh/ -[Flickr]: http://www.flickr.com/photos/sjl7678/ -[BitBucket]: http://bitbucket.org/sjl/ -[GitHub]: http://github.com/sjl/ -[BrightKite]: http://brightkite.com/people/stevelosh/ -[DjangoPeople]: http://djangopeople.net/stevelosh/ - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda content/resume/index.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/content/resume/index.markdown Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,216 @@ ++++ +date = "2010-09-20T12:34:11Z" +draft = false +title = "Résumé" + ++++ + +I'm Steve. I'm a programmer from Rochester, NY. I graduated from [Rochester +Institute of Technology][rit] in 2008 with a Bachelor's degree in [Computer +Science][ritcs]. + +I'm currently most interested in rapid web development, version control, and +artificial intelligence. I'm not looking for full time work at the moment but +I'm available for interesting freelance projects. + +If you'd *really* like to get to know me you should look at the [projects][] +and [blog posts][] I've written, or cut to the chase and look at my [code][]. + +[rit]: http://rit.edu/ +[ritcs]: http://www.cs.rit.edu/ +[projects]: /projects/ +[blog posts]: /blog/ +[code]: http://bitbucket.org/sjl/ + +## Skills & Interests + +I'm passionate about programming. It may sound odd, but there are aspects of +programming that I find beautiful (in every sense of the word). + +### Languages + +My favorite programming language at the moment is [Python][]. It's elegant, +readable, powerful, and makes it easy to *get things done*. + +I've also had experience with [Java][], [C][], [C++][], [Lisp][], [SQL][] and +[PL/SQL][], [JavaScript][], [Groovy][], and [bash scripting][]. I've used a +number of markup and templating languages, including [XHTML][], [XML][], +[CSS][], [JSON][], [Markdown][], [Django][]'s template system, and [Jinja2][]. + +In my free time I'm working on learning [R][]. + +[Python]: http://www.python.org/ +[Java]: http://java.sun.com/ +[C]: http://en.wikipedia.org/wiki/C_(programming_language) +[C++]: http://en.wikipedia.org/wiki/C%2B%2B +[Lisp]: http://en.wikipedia.org/wiki/Lisp_(programming_language) +[SQL]: http://en.wikipedia.org/wiki/SQL +[PL/SQL]: http://en.wikipedia.org/wiki/PL/SQL +[JavaScript]: http://en.wikipedia.org/wiki/JavaScript +[Groovy]: http://groovy.codehaus.org/ +[bash scripting]: http://www.gnu.org/software/bash/ +[XHTML]: http://www.w3.org/TR/xhtml1/ +[XML]: http://www.w3.org/XML/ +[CSS]: http://www.w3.org/TR/CSS/ +[JSON]: http://www.json.org/ +[Markdown]: {{links.markdown}} +[Jinja2]: http://jinja.pocoo.org/2/ +[R]: http://www.r-project.org/ + +### Web Development + +For web development I gravitate toward the [Django][] framework. Django bills +itself as "The Web framework for professionals with deadlines" and this +strikes a chord with me. + +I've used the [CherryPy][] framework for smaller (but still dynamic) projects +that don't need the additional functionality of Django. + +I hate writing repetitive, difficult-to-maintain code. Even when writing +completely static websites I prefer to take advantage of templating languages. +To do that I use [Hyde][] and [Blatter][], both of which generate static HTML +from templates. + +I've used several CSS frameworks including [Blueprint][], [Tripoli][] and +[aardvark legs][]. Of these I now prefer aardvark legs because it makes +setting up a beautiful vertical rhythm simple and doesn't impose itself on the +horizontal layout. + +When writing [JavaScript][] I use [jQuery][] to eliminate a lot of the +tediousness and create elegant code. + +[Django]: {{links.django}} +[CherryPy]: {{links.cherrypy}} +[Blueprint]: http://www.blueprintcss.org/ +[Tripoli]: http://devkick.com/lab/tripoli/ +[aardvark legs]: {{links.aardvarklegs}} +[Hyde]: http://github.com/lakshmivyas/hyde +[Blatter]: {{links.blatter}} +[jQuery]: http://jquery.com/ + +### Version Control + +I think version control is essential for a project of almost any size, and +lately the concepts behind it have captured my interest. + +In my personal work I use [Mercurial][] (usually with [BitBucket][]). I've +[contributed][] several minor patches to the Mercurial core and written two +extensions: [hg-prompt][] and [hg-paste][]. + +I know my way around [Subversion][], [CVS][], and [git][] as well. + +[Mercurial]: {{links.mercurial}} +[BitBucket]: http://bitbucket.org/ +[Subversion]: http://subversion.tigris.org/ +[CVS]: http://www.nongnu.org/cvs/ +[git]: http://git-scm.com/ +[contributed]: http://selenic.com/repo/hg/log?rev=Steve+Losh +[hg-prompt]: /projects/hg-prompt/ +[hg-paste]: /projects/hg-paste/ + +### How I Work + +For my personal work I use [Mac OS X][]. I'm comfortable in [Linux][] and can +deal with [Windows][] (though I don't enjoy it). + +I use [TextMate][] and [vim][] for coding, [CSSEdit][] when designing, +[Mercurial][] for version control, [t][] to manage my personal tasks, and +deploy through [SSH][] with [fabric][]. + +[Mac OS X]: http://www.apple.com/macosx/ +[TextMate]: http://macromates.com/ +[SSH]: http://en.wikipedia.org/wiki/Secure_Shell +[fabric]: {{links.fabric}} +[Linux]: http://www.linux.org/ +[Windows]: http://www.microsoft.com/WINDOWS/ +[vim]: http://www.vim.org/ +[CSSEdit]: http://macrabbit.com/cssedit/ +[t]: /projects/t/ + +## Personal Projects + +You can find some of my pet projects on the [projects][] page. + +Lately I've been working on a few new things that aren't quite ready for +general use, but are getting close: + +* [hg-review][]: a code-review extension for Mercurial. +* [LindyHub][]: a site like BitBucket or GitHub for dancers. +* [gorilla][]: a packaging system for Python. +* [tinpan][]: a language-agnostic, vcs-agnostic continuous integration system. + +[hg-review]: http://bitbucket.org/sjl/hg-review/ +[LindyHub]: http://test.lindyhub.com/ +[gorilla]: http://bitbucket.org/sjl/gorilla/ +[tinpan]: http://bitbucket.org/sjl/tinpan/ + +## Full-Time Work Experience + +I completed two six-month [co-ops][] while at [RIT][]. Since I graduated I've +worked full time for about a year and a half. + +[co-ops]: https://www.rit.edu/about/coop_careers.html +[RIT]: http://rit.edu/ + +### Senior Software Engineer at [Dumbwaiter Design][DWD] + +*Henrietta, NY since January 2010.* + +At [Dumbwaiter][DWD] I work with our amazing designers to create beautiful websites backed by [Django][] and [Python][]. + +[DWD]: http://dwaiter.com/ + +### Software Engineer at [PAETEC][] + +*Fairport, NY from June 2008 to January 2010.* + +At [PAETEC][] I maintained and implemented new features for web applications +in [Java][] using the [Oracle E-Business Suite][oebs] and worked on underlying +[SQL][] and [PL/SQL][] code for our databases. + +[PAETEC]: http://paetec.com/ +[oebs]: http://www.oracle.com/applications/e-business-suite.html + +### Data Architecture Co-op at [Excellus BlueCross BlueShield][] + +*Rochester, NY from June 2007 to November 2007.* + +At [Excellus][] I developed software in [Java][] to interact with and manage +databases of customer and provider information stored on a mainframe. I also +created and updated [JUnit][] and [Jemmy][] tests for this software. + +[Excellus BlueCross BlueShield]: https://www.excellusbcbs.com/ +[Excellus]: https://www.excellusbcbs.com/ +[JUnit]: http://www.junit.org/ +[Jemmy]: http://jemmy.dev.java.net/ + +### Managed Services Programmer at [RightNow Technologies][] + +*Pittsford, NY from June 2006 to November 2006.* + +At [RightNow][] I developed a set of tools to test the effectiveness of +voice-automated telephone systems. These tools were written in a combination +of [bash scripting][] and [Python][]. I assisted in generating statistics +about the effectiveness of these systems for customers. + +[RightNow Technologies]: http://www.rightnow.com/ +[RightNow]: http://www.rightnow.com/ + +## Contact Me + +If you'd like to get in touch with me, you can use any method you like. + +My **email address** is: +My **phone number** is: (570) 417-1392 +My **skype username** is: steve.losh +My **mailing address** is: 150 Park Avenue, Apartment 7, Rochester, NY 14607 + +You can also find me on [Twitter][], [Flickr][], [BitBucket][], [GitHub][], +[BrightKite][], or [DjangoPeople][]. + +[Twitter]: http://twitter.com/stevelosh/ +[Flickr]: http://www.flickr.com/photos/sjl7678/ +[BitBucket]: http://bitbucket.org/sjl/ +[GitHub]: http://github.com/sjl/ +[BrightKite]: http://brightkite.com/people/stevelosh/ +[DjangoPeople]: http://djangopeople.net/stevelosh/ diff -r bbf39c61e3fe -r e7bc59b9ebda custom/__init__.py diff -r bbf39c61e3fe -r e7bc59b9ebda custom/processors.py --- a/custom/processors.py Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,22 +0,0 @@ -import os -import sys -from django.conf import settings -from hydeengine.file_system import File -from subprocess import check_output, CalledProcessError - -class Wisp: - @staticmethod - def process(resource): - wisp = settings.WISP_PATH - if not wisp or not os.path.exists(wisp): - raise ValueError("Wisp cannot be found at [%s]" % wisp) - - try: - data = check_output([wisp, "-c", resource.source_file.path]).decode('utf-8') - except CalledProcessError, e: - print 'Syntax Error when calling Wisp:', e - return None - - out_file = File(resource.source_file.path_without_extension + ".js") - resource.source_file = out_file - resource.source_file.write(data) diff -r bbf39c61e3fe -r e7bc59b9ebda custom/templatetags/__init__.py diff -r bbf39c61e3fe -r e7bc59b9ebda custom/templatetags/mathjax.py --- a/custom/templatetags/mathjax.py Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,9 +0,0 @@ -from django import template - -register = template.Library() - - -def mathjax(value): - return "
$$ " + value + " $$
" - -register.filter('mathjax', mathjax) diff -r bbf39c61e3fe -r e7bc59b9ebda layout/_flatpage.html --- a/layout/_flatpage.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,13 +0,0 @@ -{% extends "skeleton/_base.html" %} - -{% block content %} -

{{ page.title|safe|typogrify }}

- -
- {% filter typogrify %} - {% markdown toc def_list %} - {% block article %}{% endblock %} - {% endmarkdown %} - {% endfilter %} -
-{% endblock %} \ No newline at end of file diff -r bbf39c61e3fe -r e7bc59b9ebda layout/_post.html --- a/layout/_post.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,36 +0,0 @@ -{% extends "skeleton/_base.html" %} - -{% block extra_css %} - -{% endblock %} - -{% block content %} - - - -
-

- Posted - - - on {{ page.created|date:"F j, Y" }}. -

-
- -
- {% filter typogrify %} - {% article %} - {% filter typogrify %} - {% markdown toc def_list codehilite %} - {% block article %}{% endblock %} - {% endmarkdown %} - {% endfilter %} - {% endarticle %} - {% endfilter %} -
-{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda layout/_splash.html --- a/layout/_splash.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,28 +0,0 @@ -{% extends "skeleton/_base.html" %} - -{% block title %}{{ site.name }}{% endblock %} - -{% block content %} -
- {% filter typogrify %} -

- Hello, - I'm - Steve Losh. -

- -

- I'm a - programmer, photographer, dancer & bassist - currently living in - Reykjavík, Iceland. -

- -

- If you want to get in touch with me, - - is best. -

- {% endfilter %} -
-{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda layout/skeleton/_atom.xml --- a/layout/skeleton/_atom.xml Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,44 +0,0 @@ - - -{% spaceless %} - - - {% block title %}{{ site.name }}{% endblock title %} - - {% block self_url %} - - {% endblock %} - - {% block site_url %} - - {% endblock %} - - {% block feed_extra %}{% endblock %} - - {{ now|xmldatetime }} - - {{ site.full_url }}/ - - {% recent_posts recents 4 page.blog_node %} - - {% for post in recents %} - {% if not post.listing and not post.exclude %} - - {{ post.title }} - {{ site.author }} - - {{ post.created|xmldatetime }} - {{ post.created|xmldatetime }} - {{ post.full_url }} - {% block entry_extra %}{% endblock %} - - {% filter force_escape %} - {% render_article post %} - {% endfilter %} - - - {% endif %} - {% endfor %} - - -{% endspaceless %} diff -r bbf39c61e3fe -r e7bc59b9ebda layout/skeleton/_base.html --- a/layout/skeleton/_base.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,98 +0,0 @@ -{% extends "skeleton/_root.html" %} -{% block all %} - - - - - - - - {% block title %}{% block extra_title %}{% endblock %}{{ page.title|safe }} / {{ site.name }}{% endblock %} - - {% block feeds %} - - {% endblock %} - - {% block css %} - - - - {% block extra_css %}{% endblock %} - {% endblock %} - - {% block js %} - - - - - - - {% block extra_js %}{% endblock %} - {% endblock %} - - - - - - - - - -
-
-
- steve losh -
- - -
- -
 
- -
- {% with page.node.ancestors|last as parent_node %} - {% with parent_node.url as parent_url %} - {% block content %}{% endblock %} - {% endwith %} - {% endwith %} -
- -
 
- - -
- - -{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda layout/skeleton/_body.html --- a/layout/skeleton/_body.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,7 +0,0 @@ -{% extends "skeleton/_base.html" %} - -{% block title %}{{site.name}} / {{ page.title }}{% endblock %} - -{% block content %} - {% block content_body %}{% endblock %} -{% endblock %} \ No newline at end of file diff -r bbf39c61e3fe -r e7bc59b9ebda layout/skeleton/_index.html --- a/layout/skeleton/_index.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,7 +0,0 @@ -{% extends "skeleton/_base.html" %} - -{% block content %} - {% for node in page.node.walk %} - {% include "skeleton/_innerindex.html" %} - {% endfor %} -{% endblock %} \ No newline at end of file diff -r bbf39c61e3fe -r e7bc59b9ebda layout/skeleton/_innerindex.html --- a/layout/skeleton/_innerindex.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,27 +0,0 @@ -{% extends "skeleton/_root.html" %} - -{% block all %} -{%spaceless%} -{% for list_page in node.pages %} - {% ifnotequal list_page node.listing_page %} - {% if not list_page.exclude %} -
-
- {% with list_page.name_without_extension|remove_date_prefix|unslugify as default_title %} -

- - {{ list_page.title|default_if_none:default_title }} - -

- {% if list_page.created %} - {{ list_page.created }} - {% endif %} - {%endwith%} -
- {% render_article list_page %} -
- {%endif%} - {% endifnotequal %} -{% endfor %} -{%endspaceless%} -{% endblock %} \ No newline at end of file diff -r bbf39c61e3fe -r e7bc59b9ebda layout/skeleton/_innerlisting.html --- a/layout/skeleton/_innerlisting.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,15 +0,0 @@ -{% extends "skeleton/_root.html" %} - -{% block all %} -{%spaceless%} - -{% for list_page in node.pages %} - {% ifnotequal list_page node.listing_page %} - {% if not list_page.exclude %} - {% include "skeleton/_listingitem.html" %} - {%endif%} - {% endifnotequal %} -{% endfor %} - -{%endspaceless%} -{% endblock %} \ No newline at end of file diff -r bbf39c61e3fe -r e7bc59b9ebda layout/skeleton/_listing.html --- a/layout/skeleton/_listing.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,11 +0,0 @@ -{% extends "skeleton/_base.html"%} - -{% block content %} -
-
    - {% for node in page.node.walk reversed %} - {% include "skeleton/_innerlisting.html" %} - {% endfor %} -
-
-{% endblock %} diff -r bbf39c61e3fe -r e7bc59b9ebda layout/skeleton/_listingitem.html --- a/layout/skeleton/_listingitem.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,8 +0,0 @@ -
  • - - {% with list_page.name_without_extension|remove_date_prefix|unslugify as default_title %} - {{ list_page.title|default_if_none:default_title|safe|typogrify }} - {%endwith%} - - {{ list_page.snip|safe|typogrify }} -
  • diff -r bbf39c61e3fe -r e7bc59b9ebda layout/skeleton/_root.html --- a/layout/skeleton/_root.html Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1 +0,0 @@ -{% block all %}{% endblock %} \ No newline at end of file diff -r bbf39c61e3fe -r e7bc59b9ebda layouts/shortcodes/toc.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/layouts/shortcodes/toc.html Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,1 @@ +{{ .Page.TableOfContents }} diff -r bbf39c61e3fe -r e7bc59b9ebda media/css/aal.css --- a/media/css/aal.css Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,115 +0,0 @@ -/* - aardvark.legs originally by Anatoli Papirovski - http://fecklessmind.com/ - Licensed under the MIT license. http://www.opensource.org/licenses/mit-license.php -*/ - -/* - Reset first. Modified version of Eric Meyer and Paul Chaplin reset - from http://meyerweb.com/eric/tools/css/reset/ -*/ -html, body, div, span, applet, object, iframe, -h1, h2, h3, h4, h5, h6, p, blockquote, pre, -a, abbr, acronym, address, big, cite, code, -del, dfn, em, font, img, ins, kbd, q, s, samp, -small, strike, strong, sub, sup, tt, var, -b, u, i, center, -dl, dt, dd, ol, ul, li, -fieldset, form, label, legend, -table, caption, tbody, tfoot, thead, tr, th, td, -header, nav, section, article, aside, footer -{border: 0; margin: 0; outline: 0; padding: 0; background: transparent; vertical-align: baseline;} - -article,aside,details,figcaption,figure,footer,header,hgroup,menu,nav,section { - display:block; -} - -blockquote, q {quotes: none;} -blockquote:before,blockquote:after,q:before,q:after {content: ''; content: none;} - -header, nav, section, article, aside, footer {display: block;} - -/* Basic styles */ -html {overflow-y: scroll;} -body {background: #fdfdfd; color: #353535; font: normal 18px/25px Palatino, "Palatino Linotype", serif; text-rendering: optimizeLegibility;} -html>body {font-size: 18px; line-height: 25px;} - -img {display: inline-block; vertical-align: bottom;} - -h1,h2,h3,h4,h5,h6,strong,b,dt,th {font-weight: 700;} -address,cite,em,i,caption,dfn,var {font-style: italic;} - -h1 { font-size: 45px; line-height: 50px; margin: 25px 0; } -h2 { font-size: 32px; line-height: 50px; margin: 25px 0; } -h3 { font-size: 23px; line-height: 25px; margin: 25px 0; } -h4 {margin: 0 0 22px; font-size: 16px; line-height: 22px;} -h5 {margin: 0 0 22px; font-size: 14px; line-height: 22px;} -h6 {margin: 0 0 22px; font-size: 12px; line-height: 22px;} - -p,ul,ol,dl,blockquote,pre {margin: 0 0 25px;} - -li ul,li ol {margin: 0;} -ul {list-style: outside disc;} -ol {list-style: outside decimal;} -li {margin: 0 0 0 44px;} -dd {padding-left: 25px;} -blockquote {padding: 0 25px;} - -a {text-decoration: underline;} -a:hover {text-decoration: none;} -abbr,acronym {border-bottom: 1px dotted; cursor: help;} -del {text-decoration: line-through;} -ins {text-decoration: overline;} -sub {font-size: 14px; line-height: 25px; vertical-align: sub;} -sup {font-size: 14px; line-height: 25px; vertical-align: super;} - -tt,code,kbd,samp,pre {font-size: 14px; line-height: 25px; font-family: Menlo, Monaco, Consolas, "Courier New", monospace;} - -/* Table styles */ -/* TODO */ -table {border-collapse: collapse; border-spacing: 0; margin: 0 0 1.5em;} -caption {text-align: left;} -th, td {padding: .25em .5em;} -tbody td, tbody th {border: 1px solid #222;} -tfoot {font-style: italic;} - -/* Form styles */ -/* TODO */ -fieldset {clear: both;} -legend {padding: 0 0 1.286em; font-size: 1.167em; font-weight: 700;} -fieldset fieldset legend {padding: 0 0 1.5em; font-size: 1em;} -* html legend {margin-left: -7px;} -*+html legend {margin-left: -7px;} - -form .field, form .buttons {clear: both; margin: 0 0 1.5em;} -form .field label {display: block;} -form ul.fields li {list-style-type: none; margin: 0;} -form ul.inline li, form ul.inline label {display: inline;} -form ul.inline li {padding: 0 .75em 0 0;} - -input.radio, input.checkbox {vertical-align: top;} -label, button, input.submit, input.image {cursor: pointer;} -* html input.radio, * html input.checkbox {vertical-align: middle;} -*+html input.radio, *+html input.checkbox {vertical-align: middle;} - -textarea {overflow: auto;} -input.text, input.password, textarea, select {margin: 0; font: 1em/1.3 Helvetica, Arial, "Liberation Sans", "Bitstream Vera Sans", sans-serif; vertical-align: baseline;} -input.text, input.password, textarea {border: 1px solid #444; border-bottom-color: #666; border-right-color: #666; padding: 2px;} - -* html button {margin: 0 .34em 0 0;} -*+html button {margin: 0 .34em 0 0;} - -form.horizontal .field {padding-left: 150px;} -form.horizontal .field label {display: inline; float: left; width: 140px; margin-left: -150px;} - -/* Useful classes */ -/* TODO */ -img.left {display: inline; float: left; margin: 0 1.5em .75em 0;} -img.right {display: inline; float: right; margin: 0 0 .75em .75em;} -.group:after { - content: "."; - display: block; - height: 0; - clear: both; - visibility: hidden; -} - diff -r bbf39c61e3fe -r e7bc59b9ebda media/css/print.less --- a/media/css/print.less Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,68 +0,0 @@ -body { - font: normal 10pt/1.25 Palatino, "Palatino Linotype"; - text-rendering: optimizeLegibility; -} - -nav, header, .toc { - display: none; -} -#leaf-title { - a { - text-decoration: none; - color: black; - } -} -#leaf-content { - ul.print-links { - display: block; - font-size: 1em; - list-style-type: none; - margin-left: 0em; - - a { - text-decoration: none; - } - } - code, pre { - font: normal 9pt Menlo, Monaco, Consolas, "Courier New", Courier, monospace; - } - img { - display: block; - margin-left: auto; - margin-right: auto; - border: 1.43em solid #e5e5e5; - padding: 1px; - background: black; - } - img.left, img.right { - border: none; - background: none; - padding: none; - } - img.left { - margin: 0 1.5em 1em 0; - float: left; - } - img.right { - margin: 0 0 .75em 1em; - float: right; - } -} -div#leaf-content.with-diagrams img { - display: block; - margin-left: auto; - margin-right: auto; - background: none; - border: none; -} -span.amp { - font-family: "Palatino", "Constantia", "Palatino Linotype", serif; - font-style: italic; -} -a { - text-decoration: underline; - color: #c06; -} -footer { - display: none; -} diff -r bbf39c61e3fe -r e7bc59b9ebda media/css/pygments-clean.css --- a/media/css/pygments-clean.css Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,60 +0,0 @@ -/* @override http://localhost:8080/media/css/pygments-monokai-light.css */ -.codehilite .hll { background-color: #49483e } -.codehilite .err { color: #fff; background-color: #f00 } /* Error */ -.codehilite .k { color: #111} /* Keyword */ -.codehilite .l { color: #111 } /* Literal */ -.codehilite .n { color: #111 } /* Name */ -.codehilite .o { color: #111 } /* Operator */ -.codehilite .p { color: #111 } /* Punctuation */ -.codehilite .c { color: #714678; font-style: italic; font-weight: bold; } /* Comment */ -.codehilite .cm { color: #714678; font-style: italic; font-weight: bold; } /* Comment.Multiline */ -.codehilite .cp { color: #714678; font-style: italic; font-weight: bold; } /* Comment.Preproc */ -.codehilite .c1 { color: #714678; font-style: italic; font-weight: bold; } /* Comment.Single */ -.codehilite .cs { color: #714678; font-style: italic; font-weight: bold; } /* Comment.Special */ -.codehilite .ge { font-style: italic } /* Generic.Emph */ -.codehilite .gs { font-weight: bold } /* Generic.Strong */ -.codehilite .kc { color: #111 } /* Keyword.Constant */ -.codehilite .kd { color: #111 } /* Keyword.Declaration */ -.codehilite .kn { color: #111 } /* Keyword.Namespace */ -.codehilite .kp { color: #111 } /* Keyword.Pseudo */ -.codehilite .kr { color: #111 } /* Keyword.Reserved */ -.codehilite .kt { color: #111 } /* Keyword.Type */ -.codehilite .ld { color: #111 } /* Literal.Date */ -.codehilite .m { color: #111 } /* Literal.Number */ -.codehilite .s { color: #111} /* Literal.String */ -.codehilite .na { color: #111 } /* Name.Attribute */ -.codehilite .nb { color: #111 } /* Name.Builtin */ -.codehilite .nc { color: #111 } /* Name.Class */ -.codehilite .no { color: #111 } /* Name.Constant */ -.codehilite .nd { color: #111 } /* Name.Decorator */ -.codehilite .ni { color: #111 } /* Name.Entity */ -.codehilite .ne { color: #111 } /* Name.Exception */ -.codehilite .nf { color: #111} /* Name.Function */ -.codehilite .nl { color: #111 } /* Name.Label */ -.codehilite .nn { color: #111} /* Name.Namespace */ -.codehilite .nx { color: #111 } /* Name.Other */ -.codehilite .py { color: #111 } /* Name.Property */ -.codehilite .nt { color: #111 } /* Name.Tag */ -.codehilite .nv { color: #111 } /* Name.Variable */ -.codehilite .ow { color: #111 } /* Operator.Word */ -.codehilite .w { color: #111 } /* Text.Whitespace */ -.codehilite .mf { color: #111 } /* Literal.Number.Float */ -.codehilite .mh { color: #111 } /* Literal.Number.Hex */ -.codehilite .mi { color: #111 } /* Literal.Number.Integer */ -.codehilite .mo { color: #111 } /* Literal.Number.Oct */ -.codehilite .sb { color: #111 } /* Literal.String.Backtick */ -.codehilite .sc { color: #111 } /* Literal.String.Char */ -.codehilite .sd { color: #111 } /* Literal.String.Doc */ -.codehilite .s2 { color: #111 } /* Literal.String.Double */ -.codehilite .se { color: #111 } /* Literal.String.Escape */ -.codehilite .sh { color: #111 } /* Literal.String.Heredoc */ -.codehilite .si { color: #111 } /* Literal.String.Interpol */ -.codehilite .sx { color: #111 } /* Literal.String.Other */ -.codehilite .sr { color: #111 } /* Literal.String.Regex */ -.codehilite .s1 { color: #111 } /* Literal.String.Single */ -.codehilite .ss { color: #111 } /* Literal.String.Symbol */ -.codehilite .bp { color: #111 } /* Name.Builtin.Pseudo */ -.codehilite .vc { color: #111 } /* Name.Variable.Class */ -.codehilite .vg { color: #111 } /* Name.Variable.Global */ -.codehilite .vi { color: #111 } /* Name.Variable.Instance */ -.codehilite .il { color: #111 } /* Literal.Number.Integer.Long */ diff -r bbf39c61e3fe -r e7bc59b9ebda media/css/pygments-monokai-light.css --- a/media/css/pygments-monokai-light.css Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,60 +0,0 @@ -/* @override http://localhost:8080/media/css/pygments-monokai-light.css */ -.codehilite .hll { background-color: #49483e } -.codehilite .c { color: #7A7663 } /* Comment */ -.codehilite .err { color: #960050; background-color: #1e0010 } /* Error */ -.codehilite .k { color: #00a8c8} /* Keyword */ -.codehilite .l { color: #ae81ff } /* Literal */ -.codehilite .n { color: #111111 } /* Name */ -.codehilite .o { color: #f92672 } /* Operator */ -.codehilite .p { color: #111111 } /* Punctuation */ -.codehilite .cm { color: #75715e } /* Comment.Multiline */ -.codehilite .cp { color: #75715e } /* Comment.Preproc */ -.codehilite .c1 { color: #75715e } /* Comment.Single */ -.codehilite .cs { color: #75715e } /* Comment.Special */ -.codehilite .ge { font-style: italic } /* Generic.Emph */ -.codehilite .gs { font-weight: bold } /* Generic.Strong */ -.codehilite .kc { color: #00a8c8 } /* Keyword.Constant */ -.codehilite .kd { color: #00a8c8 } /* Keyword.Declaration */ -.codehilite .kn { color: #f92672 } /* Keyword.Namespace */ -.codehilite .kp { color: #00a8c8 } /* Keyword.Pseudo */ -.codehilite .kr { color: #00a8c8 } /* Keyword.Reserved */ -.codehilite .kt { color: #00a8c8 } /* Keyword.Type */ -.codehilite .ld { color: #d88200 } /* Literal.Date */ -.codehilite .m { color: #ae81ff } /* Literal.Number */ -.codehilite .s { color: #d88200} /* Literal.String */ -.codehilite .na { color: #75af00 } /* Name.Attribute */ -.codehilite .nb { color: #111111 } /* Name.Builtin */ -.codehilite .nc { color: #75af00 } /* Name.Class */ -.codehilite .no { color: #00a8c8 } /* Name.Constant */ -.codehilite .nd { color: #75af00 } /* Name.Decorator */ -.codehilite .ni { color: #111111 } /* Name.Entity */ -.codehilite .ne { color: #75af00 } /* Name.Exception */ -.codehilite .nf { color: #75af00} /* Name.Function */ -.codehilite .nl { color: #111111 } /* Name.Label */ -.codehilite .nn { color: #111111} /* Name.Namespace */ -.codehilite .nx { color: #111111 } /* Name.Other */ -.codehilite .py { color: #111111 } /* Name.Property */ -.codehilite .nt { color: #f92672 } /* Name.Tag */ -.codehilite .nv { color: #111111 } /* Name.Variable */ -.codehilite .ow { color: #f92672 } /* Operator.Word */ -.codehilite .w { color: #111111 } /* Text.Whitespace */ -.codehilite .mf { color: #ae81ff } /* Literal.Number.Float */ -.codehilite .mh { color: #ae81ff } /* Literal.Number.Hex */ -.codehilite .mi { color: #ae81ff } /* Literal.Number.Integer */ -.codehilite .mo { color: #ae81ff } /* Literal.Number.Oct */ -.codehilite .sb { color: #d88200 } /* Literal.String.Backtick */ -.codehilite .sc { color: #d88200 } /* Literal.String.Char */ -.codehilite .sd { color: #d88200 } /* Literal.String.Doc */ -.codehilite .s2 { color: #d88200 } /* Literal.String.Double */ -.codehilite .se { color: #8045FF } /* Literal.String.Escape */ -.codehilite .sh { color: #d88200 } /* Literal.String.Heredoc */ -.codehilite .si { color: #d88200 } /* Literal.String.Interpol */ -.codehilite .sx { color: #d88200 } /* Literal.String.Other */ -.codehilite .sr { color: #d88200 } /* Literal.String.Regex */ -.codehilite .s1 { color: #d88200 } /* Literal.String.Single */ -.codehilite .ss { color: #d88200 } /* Literal.String.Symbol */ -.codehilite .bp { color: #111111 } /* Name.Builtin.Pseudo */ -.codehilite .vc { color: #111111 } /* Name.Variable.Class */ -.codehilite .vg { color: #111111 } /* Name.Variable.Global */ -.codehilite .vi { color: #111111 } /* Name.Variable.Instance */ -.codehilite .il { color: #ae81ff } /* Literal.Number.Integer.Long */ diff -r bbf39c61e3fe -r e7bc59b9ebda media/css/sjl.less --- a/media/css/sjl.less Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,362 +0,0 @@ -@c-link: #e50053; -@c-body: #222; -@w-body: 612px; -@font-fancy: 'OFL Sorts Mill Goudy', 'OFL Sorts Mill Goudy TT', serif; - -html { - overflow-y: scroll; -} -body { - text-rendering: optimizeLegibility; - color: @c-body; - position: relative; - - a { - text-decoration: none; - color: @c-link; - - &:hover { - text-decoration: underline; - } - } - h1, h2, h3, h4, h5, h6 { - font-family: @font-fancy; - font-weight: normal; - - a { - color: @c-body; - - &:hover { - color: @c-link; - text-decoration: none; - } - } - .amp { - font-style: italic; - } - } - h1 { font-size: 45px; line-height: 50px; margin: 25px 0; } // 3 - h2 { font-size: 32px; line-height: 50px; margin: 25px 0; } // m7 - h3 { font-size: 23px; line-height: 25px; margin: 25px 0; } // 3 - h4 { font-size: 18px; line-height: 25px; margin: 25px 0; font-weight: bold; } // r - code, pre { - font-family: Consolas, Menlo, "Courier New", monospace; - font-size: 14px; - } - div.threejs { - margin-bottom: 16px; - - canvas { - border: 1px solid #222222; - } - div { - text-align: center; - } - } - .codehilite { - code, pre { - font-family: Consolas, Menlo, "Courier New", monospace; - font-size: 16px; - line-height: 25px; - - overflow-x: auto; - - border: 1px solid #d5d5d5; - border-left: 10px solid #d5d5d5; - background-color: #fafafa; - - padding: 11px 15px 12px; - margin-left: -25px; - } - pre::-webkit-scrollbar { - height: 25px; - } - pre::-webkit-scrollbar-button:start, - pre::-webkit-scrollbar-button:end { - display: none; - } - pre::-webkit-scrollbar-track-piece { - background-color: #eee; - } - pre::-webkit-scrollbar-thumb { - background-color: #bbb; - border: 7px solid #eee; - -webkit-background-clip: padding-box; - -webkit-border-radius: 12px; - } - } - pre.lineart { - font-family: Consolas, Menlo, "Courier New", monospace; - font-size: 16px; - line-height: 20px; - border: none; - } - p code, li code, table code { - border: 1px solid #ccc; - background-color: #fafafa; - font-size: 13px; - padding: 1px 3px; - line-height: 20px; - margin: 0; - white-space: nowrap; - } - - .wrap { - width: @w-body; - margin: 0 auto; - margin-bottom: 5em; - - .top { - header { - float: left; - font-family: @font-fancy; - font-size: 23px; // 3 - line-height: 50px; - text-transform: lowercase; - padding-left: 2px; - width: 612px - 2px - 400px; - - .amp { - font-style: italic; - } - a { - color: @c-body; - - &:hover { - color: @c-link; - } - } - } - nav { - font: normal 18px/50px @font-fancy; // 1 - text-align: right; - text-transform: lowercase; - padding-right: 2px; - width: 400px - 2px; - float: left; - - .sep { - padding: 0 4px; - color: #666; - } - } - } - .hr { - margin-top: -12px; - margin-bottom: 12px + 25px; - height: 25px; - background: transparent url('/media/images/hr.png') top left no-repeat; - } - .hrb { - height: 25px; - margin-top: 50px; - background: transparent url('/media/images/hrb.png') top left no-repeat; - } - footer { - text-align: center; - - p { - font-size: 14px; - font-style: italic; - line-height: 50px; - margin-bottom: 0px; - } - .rochester-made { - img { - opacity: 0.7; - padding: 5px 20px; - } - } - } - } -} -hr { - border: none; - background: #ccc; - height: 1px; - margin-bottom: 24px; -} -.splash { - @color: #454545; - color: @color; - text-align: center; - font: normal 27px/32px @font-fancy; - margin-bottom: -1px; - padding-top: 0; - - p { - margin-bottom: 24px; - } - .amp { - font-style: italic; - } - .fn { - color: @color; - text-decoration: none; - - &:hover { - color: @c-link; - text-decoration: none; - } - .last-name { - display: none; - } - } -} -.section-listing { - margin-bottom: -50px; - ol { - list-style-type: none; - - @w-listing-item: 612px/2; - @w-listing-col-padding: 50px; - - li { - margin: 0 0 25px 0; - - a { - font: normal 23px/32px @font-fancy; // 3 - color: @c-body; - display: block; - - &:hover { - color: @c-link; - text-decoration: none; - } - } - - span.snip { - font-size: 18px; // 1 - color: #333; - font-family: @font-fancy; - line-height: 25px; - font-style: italic; - } - .amp { - font-style: italic; - } - } - } -} - -blockquote { - border: 1px solid #ccc; - background-color: #fafafa; - padding: 11px 15px 12px; - margin-left: 2em; - overflow: auto; - - p:last-child { - margin-bottom: 0; - } -} -span.dquo { - margin-left: -0.23em; -} - -#leaf-stats p { - color: #666; - margin-top: -22px; - margin-bottom: 22px; -} -#leaf-content { - img { - display: block; - margin: 25px auto 26px; - border: 11px solid #e5e5e5; - padding: 1px; - background: black; - max-width: 590px; - } - img.left, img.right { - border: none; - background: none; - padding: none; - } - img.left { - margin: 0 1.5em 1em 0; - } - img.right { - margin: 0 0 .75em 1em; - } - .gallery img { - background: none; - padding: 0; - border: none; - display: inline; - margin-bottom: 25px; - margin-right: 25px; - } - .toc { - ul { - list-style: none; - } - - ul:first-child>li { - margin-left: 0em; - } - } - table { - padding: 0px; - margin-top: -8px; - margin-bottom: 25px; - - tr { - margin: 0px; - padding: 0px; - - td, th { - margin: 0px; - padding: 5px 5px; - line-height: 23px; - } - td { - border: 1px solid #666; - } - } - } -} -#leaf-content.with-diagrams img { - display: block; - margin: 25px auto; - padding: 0; - background: none; - border: none; -} - -.print-links { - display: none; -} -img.self { - border: none; - padding: 0; - margin: 0; - margin-right: -108px; - margin-top: -15px; - margin-left: 30px; - margin-bottom: 20px; -} - -div#cboxCurrent { - bottom: -30px; - font-size: 17px; - font-weight: normal; - left: 60px; -} -div.screenshots { - img { - max-width: 580px; - } -} - -#scrolling-header { - color: #999; - font-size: 23px; - font-style: italic; - font: italic 23px @font-fancy; - line-height: 30px; - position: fixed; - top: 75px; - text-align: right; - width: 180px; -} diff -r bbf39c61e3fe -r e7bc59b9ebda media/diamond-square.monopic Binary file media/diamond-square.monopic has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2009/02/dj-playlist-sorting.png Binary file media/images/blog/2009/02/dj-playlist-sorting.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2009/02/dj-playlist-unrated.png Binary file media/images/blog/2009/02/dj-playlist-unrated.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2009/03/prompt-with-branch.png Binary file media/images/blog/2009/03/prompt-with-branch.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2009/03/prompt-with-dirty.png Binary file media/images/blog/2009/03/prompt-with-dirty.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2009/03/prompt-without-branch.png Binary file media/images/blog/2009/03/prompt-without-branch.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2009/03/terminal-colors.png Binary file media/images/blog/2009/03/terminal-colors.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2009/08/branch-anon.png Binary file media/images/blog/2009/08/branch-anon.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2009/08/branch-base.png Binary file media/images/blog/2009/08/branch-base.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2009/08/branch-bookmark.png Binary file media/images/blog/2009/08/branch-bookmark.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2009/08/branch-clone.png Binary file media/images/blog/2009/08/branch-clone.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2009/08/branch-named.png Binary file media/images/blog/2009/08/branch-named.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2010/01/mercurial-vs-git.jpg Binary file media/images/blog/2010/01/mercurial-vs-git.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2010/02/hg-branching-1-after-merge.png Binary file media/images/blog/2010/02/hg-branching-1-after-merge.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2010/02/hg-branching-1-needs-merge.png Binary file media/images/blog/2010/02/hg-branching-1-needs-merge.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2010/02/hg-branching-1-other.png Binary file media/images/blog/2010/02/hg-branching-1-other.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2010/02/hg-branching-1-start.png Binary file media/images/blog/2010/02/hg-branching-1-start.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2010/02/zsh-prompt-comments.png Binary file media/images/blog/2010/02/zsh-prompt-comments.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2010/02/zsh-prompt.png Binary file media/images/blog/2010/02/zsh-prompt.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2010/05/default-stable-example.png Binary file media/images/blog/2010/05/default-stable-example.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2010/06/translation-branches.png Binary file media/images/blog/2010/06/translation-branches.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2010/08/git-basics.png Binary file media/images/blog/2010/08/git-basics.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2010/08/mercurial-basics.png Binary file media/images/blog/2010/08/mercurial-basics.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2010/08/mq-multiple.png Binary file media/images/blog/2010/08/mq-multiple.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2010/08/mq-one.png Binary file media/images/blog/2010/08/mq-one.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2010/08/mq-two.png Binary file media/images/blog/2010/08/mq-two.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2010/08/mq-versioned.png Binary file media/images/blog/2010/08/mq-versioned.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2010/09/rainbow.png Binary file media/images/blog/2010/09/rainbow.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2010/09/vim.png Binary file media/images/blog/2010/09/vim.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2011/05/rules-1-doxie.png Binary file media/images/blog/2011/05/rules-1-doxie.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2011/05/rules-2-jotnot.png Binary file media/images/blog/2011/05/rules-2-jotnot.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2011/05/rules-3-ocr.png Binary file media/images/blog/2011/05/rules-3-ocr.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2011/05/rules-4-clean.png Binary file media/images/blog/2011/05/rules-4-clean.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/07/caves-01-01.png Binary file media/images/blog/2012/07/caves-01-01.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/07/caves-02-01.png Binary file media/images/blog/2012/07/caves-02-01.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/07/caves-02-02.png Binary file media/images/blog/2012/07/caves-02-02.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/07/caves-02-03.png Binary file media/images/blog/2012/07/caves-02-03.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/07/caves-03-1-01.png Binary file media/images/blog/2012/07/caves-03-1-01.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/07/caves-03-1-02.png Binary file media/images/blog/2012/07/caves-03-1-02.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/07/caves-03-1-03.png Binary file media/images/blog/2012/07/caves-03-1-03.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/07/caves-03-2-01.png Binary file media/images/blog/2012/07/caves-03-2-01.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/07/caves-03-2-02.png Binary file media/images/blog/2012/07/caves-03-2-02.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/07/caves-03-2-03.png Binary file media/images/blog/2012/07/caves-03-2-03.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/07/caves-03-3-01.png Binary file media/images/blog/2012/07/caves-03-3-01.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/07/caves-03-3-02.png Binary file media/images/blog/2012/07/caves-03-3-02.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/07/caves-04-01.png Binary file media/images/blog/2012/07/caves-04-01.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/07/caves-04-02.png Binary file media/images/blog/2012/07/caves-04-02.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/07/caves-04-03.png Binary file media/images/blog/2012/07/caves-04-03.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/07/caves-05-01.png Binary file media/images/blog/2012/07/caves-05-01.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/07/caves-05-02.png Binary file media/images/blog/2012/07/caves-05-02.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/07/caves-05-03.png Binary file media/images/blog/2012/07/caves-05-03.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/07/caves-06-01.png Binary file media/images/blog/2012/07/caves-06-01.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/07/caves-interlude-1-01.png Binary file media/images/blog/2012/07/caves-interlude-1-01.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/07/caves-interlude-1-02.png Binary file media/images/blog/2012/07/caves-interlude-1-02.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/10/caves-07-1-1.png Binary file media/images/blog/2012/10/caves-07-1-1.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/10/caves-07-1-2.png Binary file media/images/blog/2012/10/caves-07-1-2.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/10/kb-apple.jpg Binary file media/images/blog/2012/10/kb-apple.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/10/kb-caps.png Binary file media/images/blog/2012/10/kb-caps.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/10/kb-das.jpg Binary file media/images/blog/2012/10/kb-das.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/10/kb-hex.png Binary file media/images/blog/2012/10/kb-hex.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/10/kb-hhkb.jpg Binary file media/images/blog/2012/10/kb-hhkb.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/10/kb-irc.png Binary file media/images/blog/2012/10/kb-irc.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/10/kb-key-overlaid.png Binary file media/images/blog/2012/10/kb-key-overlaid.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/10/kb-pck.png Binary file media/images/blog/2012/10/kb-pck.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/10/kb-realforce.jpg Binary file media/images/blog/2012/10/kb-realforce.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/10/kb-size.jpg Binary file media/images/blog/2012/10/kb-size.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/10/keychain-1.png Binary file media/images/blog/2012/10/keychain-1.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/10/keychain-2.png Binary file media/images/blog/2012/10/keychain-2.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/10/keychain-3.png Binary file media/images/blog/2012/10/keychain-3.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/10/mutt-attachments.png Binary file media/images/blog/2012/10/mutt-attachments.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/10/mutt-contacts-1.png Binary file media/images/blog/2012/10/mutt-contacts-1.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/10/mutt-index.png Binary file media/images/blog/2012/10/mutt-index.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/10/mutt-pager.png Binary file media/images/blog/2012/10/mutt-pager.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/10/mutt-ready-to-send.png Binary file media/images/blog/2012/10/mutt-ready-to-send.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/10/mutt-send-1.png Binary file media/images/blog/2012/10/mutt-send-1.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/10/mutt-urls.png Binary file media/images/blog/2012/10/mutt-urls.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2012/10/what-the-mutt.png Binary file media/images/blog/2012/10/what-the-mutt.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-01.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-01.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-02.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-02.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-03.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-03.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-04.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-04.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-05.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-05.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-06.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-06.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-07.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-07.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-08.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-08.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-09.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-09.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-10.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-10.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-11.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-11.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-12.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-12.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-13.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-13.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-14.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-14.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-15.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-15.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-16.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-16.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-17.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-17.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-18.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-18.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-19.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-19.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-20.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-20.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-21.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-21.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-22.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-22.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-23.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-23.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-24.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-24.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-25.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-25.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-26.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-26.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-27.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-27.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-28.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-28.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-29.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-29.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-30.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-30.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-31.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-31.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-32.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-32.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-33.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-33.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-34.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-34.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-35.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-35.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-36.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-36.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-37.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-37.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-38.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-38.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-39.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-39.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-40.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-40.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-41.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-41.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-42.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-42.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-43.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-43.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-44.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-44.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-45.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-45.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-46.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-46.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-47.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-47.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-48.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-48.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-a2540-49.jpg Binary file media/images/blog/2015/07/full/nat-geo-a2540-49.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-01.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-01.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-02.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-02.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-03.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-03.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-04.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-04.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-05.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-05.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-06.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-06.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-07.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-07.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-08.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-08.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-09.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-09.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-10.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-10.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-11.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-11.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-12.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-12.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-13.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-13.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-14.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-14.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-15.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-15.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-16.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-16.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-17.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-17.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-18.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-18.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-19.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-19.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-20.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-20.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-21.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-21.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-22.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-22.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-23.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-23.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-24.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-24.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-25.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-25.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-26.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-26.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-27.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-27.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-28.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-28.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-29.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-29.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-30.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-30.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-31.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-31.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-32.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-32.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-33.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-33.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-34.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-34.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-35.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-35.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-36.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-36.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-37.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-37.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-38.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-38.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-39.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-39.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-40.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-40.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-41.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-41.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-42.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-42.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-43.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-43.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-44.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-44.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-45.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-45.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-46.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-46.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-47.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-47.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-48.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-48.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-49.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-49.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-50.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-50.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-51.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-51.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-52.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-52.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-53.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-53.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-54.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-54.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-55.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-55.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-56.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-56.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-57.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-57.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-58.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-58.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-59.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-59.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-60.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-60.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-61.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-61.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-62.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-62.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-63.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-63.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-64.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-64.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-65.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-65.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-66.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-66.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-67.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-67.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-68.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-68.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-69.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-69.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-70.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-70.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-71.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-71.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-72.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-72.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-73.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-73.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-74.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-74.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-75.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-75.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-76.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-76.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-77.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-77.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-78.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-78.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-79.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-79.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/full/nat-geo-mc5350-80.jpg Binary file media/images/blog/2015/07/full/nat-geo-mc5350-80.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-01.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-01.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-02.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-02.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-03.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-03.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-04.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-04.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-05.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-05.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-06.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-06.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-07.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-07.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-08.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-08.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-09.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-09.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-10.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-10.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-11.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-11.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-12.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-12.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-13.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-13.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-14.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-14.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-15.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-15.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-16.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-16.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-17.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-17.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-18.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-18.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-19.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-19.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-20.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-20.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-21.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-21.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-22.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-22.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-23.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-23.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-24.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-24.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-25.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-25.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-26.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-26.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-27.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-27.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-28.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-28.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-29.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-29.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-30.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-30.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-31.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-31.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-32.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-32.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-33.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-33.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-34.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-34.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-35.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-35.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-36.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-36.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-37.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-37.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-38.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-38.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-39.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-39.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-40.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-40.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-41.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-41.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-42.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-42.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-43.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-43.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-44.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-44.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-45.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-45.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-46.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-46.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-47.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-47.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-48.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-48.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-a2540-49.jpg Binary file media/images/blog/2015/07/nat-geo-a2540-49.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-01.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-01.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-02.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-02.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-03.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-03.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-04.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-04.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-05.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-05.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-06.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-06.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-07.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-07.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-08.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-08.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-09.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-09.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-10.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-10.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-11.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-11.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-12.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-12.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-13.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-13.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-14.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-14.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-15.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-15.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-16.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-16.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-17.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-17.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-18.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-18.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-19.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-19.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-20.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-20.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-21.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-21.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-22.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-22.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-23.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-23.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-24.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-24.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-25.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-25.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-26.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-26.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-27.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-27.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-28.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-28.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-29.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-29.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-30.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-30.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-31.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-31.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-32.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-32.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-33.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-33.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-34.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-34.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-35.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-35.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-36.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-36.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-37.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-37.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-38.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-38.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-39.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-39.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-40.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-40.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-41.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-41.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-42.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-42.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-43.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-43.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-44.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-44.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-45.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-45.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-46.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-46.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-47.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-47.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-48.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-48.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-49.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-49.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-50.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-50.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-51.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-51.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-52.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-52.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-53.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-53.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-54.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-54.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-55.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-55.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-56.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-56.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-57.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-57.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-58.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-58.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-59.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-59.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-60.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-60.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-61.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-61.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-62.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-62.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-63.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-63.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-64.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-64.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-65.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-65.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-66.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-66.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-67.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-67.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-68.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-68.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-69.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-69.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-70.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-70.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-71.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-71.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-72.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-72.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-73.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-73.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-74.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-74.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-75.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-75.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-76.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-76.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-77.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-77.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-78.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-78.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-79.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-79.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/07/nat-geo-mc5350-80.jpg Binary file media/images/blog/2015/07/nat-geo-mc5350-80.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s2-ggsteve-large.png Binary file media/images/blog/2015/11/btd-s2-ggsteve-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s2-ggsteve.png Binary file media/images/blog/2015/11/btd-s2-ggsteve.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-bob-large.png Binary file media/images/blog/2015/11/btd-s3-bob-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-bob.png Binary file media/images/blog/2015/11/btd-s3-bob.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-douche-large.png Binary file media/images/blog/2015/11/btd-s3-douche-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-douche.png Binary file media/images/blog/2015/11/btd-s3-douche.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-ggsteve-large.png Binary file media/images/blog/2015/11/btd-s3-ggsteve-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-ggsteve.png Binary file media/images/blog/2015/11/btd-s3-ggsteve.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-innuendo-large.png Binary file media/images/blog/2015/11/btd-s3-innuendo-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-innuendo.png Binary file media/images/blog/2015/11/btd-s3-innuendo.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-mouthnoises-large.png Binary file media/images/blog/2015/11/btd-s3-mouthnoises-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-mouthnoises.png Binary file media/images/blog/2015/11/btd-s3-mouthnoises.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-percentile-bob-large.png Binary file media/images/blog/2015/11/btd-s3-percentile-bob-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-percentile-bob.png Binary file media/images/blog/2015/11/btd-s3-percentile-bob.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-percentile-cringe-large.png Binary file media/images/blog/2015/11/btd-s3-percentile-cringe-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-percentile-cringe.png Binary file media/images/blog/2015/11/btd-s3-percentile-cringe.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-percentile-drugs-large.png Binary file media/images/blog/2015/11/btd-s3-percentile-drugs-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-percentile-drugs.png Binary file media/images/blog/2015/11/btd-s3-percentile-drugs.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-percentile-steve-large.png Binary file media/images/blog/2015/11/btd-s3-percentile-steve-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-percentile-steve.png Binary file media/images/blog/2015/11/btd-s3-percentile-steve.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-percentile-the-large.png Binary file media/images/blog/2015/11/btd-s3-percentile-the-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-percentile-the.png Binary file media/images/blog/2015/11/btd-s3-percentile-the.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-ruined-large.png Binary file media/images/blog/2015/11/btd-s3-ruined-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-ruined.png Binary file media/images/blog/2015/11/btd-s3-ruined.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-steve-large.png Binary file media/images/blog/2015/11/btd-s3-steve-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-steve.png Binary file media/images/blog/2015/11/btd-s3-steve.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-subjects-large.png Binary file media/images/blog/2015/11/btd-s3-subjects-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-subjects.png Binary file media/images/blog/2015/11/btd-s3-subjects.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-the-large.png Binary file media/images/blog/2015/11/btd-s3-the-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-s3-the.png Binary file media/images/blog/2015/11/btd-s3-the.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-ssp-bob-large.png Binary file media/images/blog/2015/11/btd-ssp-bob-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-ssp-bob.png Binary file media/images/blog/2015/11/btd-ssp-bob.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-ssp-gg-large.png Binary file media/images/blog/2015/11/btd-ssp-gg-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-ssp-gg.png Binary file media/images/blog/2015/11/btd-ssp-gg.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-ssp-rip__devil-large.png Binary file media/images/blog/2015/11/btd-ssp-rip__devil-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-ssp-rip__devil.png Binary file media/images/blog/2015/11/btd-ssp-rip__devil.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-ssp-ruined-large.png Binary file media/images/blog/2015/11/btd-ssp-ruined-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-ssp-ruined.png Binary file media/images/blog/2015/11/btd-ssp-ruined.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-ssp-steve-large.png Binary file media/images/blog/2015/11/btd-ssp-steve-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-ssp-steve.png Binary file media/images/blog/2015/11/btd-ssp-steve.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-ssp-the-large.png Binary file media/images/blog/2015/11/btd-ssp-the-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-ssp-the.png Binary file media/images/blog/2015/11/btd-ssp-the.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-volume-comparison-large.png Binary file media/images/blog/2015/11/btd-volume-comparison-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/btd-volume-comparison.png Binary file media/images/blog/2015/11/btd-volume-comparison.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/hlw-heknew-large.png Binary file media/images/blog/2015/11/hlw-heknew-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/hlw-heknew.png Binary file media/images/blog/2015/11/hlw-heknew.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/hlw-higg-large.png Binary file media/images/blog/2015/11/hlw-higg-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/hlw-higg.png Binary file media/images/blog/2015/11/hlw-higg.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/hlw-love-large.png Binary file media/images/blog/2015/11/hlw-love-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/hlw-love.png Binary file media/images/blog/2015/11/hlw-love.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/hlw-ripdevil-large.png Binary file media/images/blog/2015/11/hlw-ripdevil-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/hlw-ripdevil.png Binary file media/images/blog/2015/11/hlw-ripdevil.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/hlw-total-large.png Binary file media/images/blog/2015/11/hlw-total-large.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/11/hlw-total.png Binary file media/images/blog/2015/11/hlw-total.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/12/silt-initial.gif Binary file media/images/blog/2015/12/silt-initial.gif has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2015/12/silt-later.gif Binary file media/images/blog/2015/12/silt-later.gif has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2016/08/aspect-flavor.png Binary file media/images/blog/2016/08/aspect-flavor.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2016/08/aspect-visible.png Binary file media/images/blog/2016/08/aspect-visible.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2016/08/bad-tiling-ds.png Binary file media/images/blog/2016/08/bad-tiling-ds.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2016/08/good-tiling-ds.png Binary file media/images/blog/2016/08/good-tiling-ds.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2016/08/silt-names.png Binary file media/images/blog/2016/08/silt-names.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2016/08/silt1-inspect.png Binary file media/images/blog/2016/08/silt1-inspect.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2016/08/silt1-terrain.png Binary file media/images/blog/2016/08/silt1-terrain.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2016/08/silt2-inspect.png Binary file media/images/blog/2016/08/silt2-inspect.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2016/08/silt2-terrain.png Binary file media/images/blog/2016/08/silt2-terrain.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/blog/2016/09/loop-macro.jpg Binary file media/images/blog/2016/09/loop-macro.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/hr.png Binary file media/images/hr.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/hrb.png Binary file media/images/hrb.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/projects/fuego/Fuego-4852.jpg Binary file media/images/projects/fuego/Fuego-4852.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/projects/fuego/Fuego-4887.jpg Binary file media/images/projects/fuego/Fuego-4887.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/projects/fuego/Fuego-4919.jpg Binary file media/images/projects/fuego/Fuego-4919.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/projects/fuego/Fuego-4941.jpg Binary file media/images/projects/fuego/Fuego-4941.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/projects/fuego/Fuego-4985.jpg Binary file media/images/projects/fuego/Fuego-4985.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/projects/fuego/Fuego-5133.jpg Binary file media/images/projects/fuego/Fuego-5133.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/projects/hg-prompt/prompt.png Binary file media/images/projects/hg-prompt/prompt.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/projects/lindyjam-com/lindyjamcom-splash.png Binary file media/images/projects/lindyjam-com/lindyjamcom-splash.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/projects/t/t-screenshot.png Binary file media/images/projects/t/t-screenshot.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/projects/women-in-water/alex-tele.jpg Binary file media/images/projects/women-in-water/alex-tele.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/projects/women-in-water/alex-wide.jpg Binary file media/images/projects/women-in-water/alex-wide.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/projects/women-in-water/anna-tele.jpg Binary file media/images/projects/women-in-water/anna-tele.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/projects/women-in-water/anna-wide.jpg Binary file media/images/projects/women-in-water/anna-wide.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/projects/women-in-water/erin-tele.jpg Binary file media/images/projects/women-in-water/erin-tele.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/projects/women-in-water/erin-wide.jpg Binary file media/images/projects/women-in-water/erin-wide.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/projects/women-in-water/leah-tele.jpg Binary file media/images/projects/women-in-water/leah-tele.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/projects/women-in-water/leah-wide.jpg Binary file media/images/projects/women-in-water/leah-wide.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/projects/women-in-water/lizza-tele.jpg Binary file media/images/projects/women-in-water/lizza-tele.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/projects/women-in-water/lizza-wide.jpg Binary file media/images/projects/women-in-water/lizza-wide.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/rhythm.png Binary file media/images/rhythm.png has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/images/self.jpg Binary file media/images/self.jpg has changed diff -r bbf39c61e3fe -r e7bc59b9ebda media/js/TrackballControls.js --- a/media/js/TrackballControls.js Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,606 +0,0 @@ - -/** - * @author Eberhard Graether / http://egraether.com/ - * @author Mark Lundin / http://mark-lundin.com - * @author Simone Manini / http://daron1337.github.io - * @author Luca Antiga / http://lantiga.github.io - */ - -THREE.TrackballControls = function ( object, domElement ) { - - var _this = this; - var STATE = { NONE: - 1, ROTATE: 0, ZOOM: 1, PAN: 2, TOUCH_ROTATE: 3, TOUCH_ZOOM_PAN: 4 }; - - this.object = object; - this.domElement = ( domElement !== undefined ) ? domElement : document; - - // API - - this.enabled = true; - - this.screen = { left: 0, top: 0, width: 0, height: 0 }; - - this.rotateSpeed = 1.0; - this.zoomSpeed = 1.2; - this.panSpeed = 0.3; - - this.noRotate = false; - this.noZoom = false; - this.noPan = false; - - this.staticMoving = false; - this.dynamicDampingFactor = 0.2; - - this.minDistance = 0; - this.maxDistance = Infinity; - - this.keys = [ 65 /*A*/, 83 /*S*/, 68 /*D*/ ]; - - // internals - - this.target = new THREE.Vector3(); - - var EPS = 0.000001; - - var lastPosition = new THREE.Vector3(); - - var _state = STATE.NONE, - _prevState = STATE.NONE, - - _eye = new THREE.Vector3(), - - _movePrev = new THREE.Vector2(), - _moveCurr = new THREE.Vector2(), - - _lastAxis = new THREE.Vector3(), - _lastAngle = 0, - - _zoomStart = new THREE.Vector2(), - _zoomEnd = new THREE.Vector2(), - - _touchZoomDistanceStart = 0, - _touchZoomDistanceEnd = 0, - - _panStart = new THREE.Vector2(), - _panEnd = new THREE.Vector2(); - - // for reset - - this.target0 = this.target.clone(); - this.position0 = this.object.position.clone(); - this.up0 = this.object.up.clone(); - - // events - - var changeEvent = { type: 'change' }; - var startEvent = { type: 'start' }; - var endEvent = { type: 'end' }; - - - // methods - - this.handleResize = function () { - - if ( this.domElement === document ) { - - this.screen.left = 0; - this.screen.top = 0; - this.screen.width = window.innerWidth; - this.screen.height = window.innerHeight; - - } else { - - var box = this.domElement.getBoundingClientRect(); - // adjustments come from similar code in the jquery offset() function - var d = this.domElement.ownerDocument.documentElement; - this.screen.left = box.left + window.pageXOffset - d.clientLeft; - this.screen.top = box.top + window.pageYOffset - d.clientTop; - this.screen.width = box.width; - this.screen.height = box.height; - - } - - }; - - this.handleEvent = function ( event ) { - - if ( typeof this[ event.type ] == 'function' ) { - - this[ event.type ]( event ); - - } - - }; - - var getMouseOnScreen = ( function () { - - var vector = new THREE.Vector2(); - - return function getMouseOnScreen( pageX, pageY ) { - - vector.set( - ( pageX - _this.screen.left ) / _this.screen.width, - ( pageY - _this.screen.top ) / _this.screen.height - ); - - return vector; - - }; - - }() ); - - var getMouseOnCircle = ( function () { - - var vector = new THREE.Vector2(); - - return function getMouseOnCircle( pageX, pageY ) { - - vector.set( - ( ( pageX - _this.screen.width * 0.5 - _this.screen.left ) / ( _this.screen.width * 0.5 ) ), - ( ( _this.screen.height + 2 * ( _this.screen.top - pageY ) ) / _this.screen.width ) // screen.width intentional - ); - - return vector; - - }; - - }() ); - - this.rotateCamera = ( function() { - - var axis = new THREE.Vector3(), - quaternion = new THREE.Quaternion(), - eyeDirection = new THREE.Vector3(), - objectUpDirection = new THREE.Vector3(), - objectSidewaysDirection = new THREE.Vector3(), - moveDirection = new THREE.Vector3(), - angle; - - return function rotateCamera() { - - moveDirection.set( _moveCurr.x - _movePrev.x, _moveCurr.y - _movePrev.y, 0 ); - angle = moveDirection.length(); - - if ( angle ) { - - _eye.copy( _this.object.position ).sub( _this.target ); - - eyeDirection.copy( _eye ).normalize(); - objectUpDirection.copy( _this.object.up ).normalize(); - objectSidewaysDirection.crossVectors( objectUpDirection, eyeDirection ).normalize(); - - objectUpDirection.setLength( _moveCurr.y - _movePrev.y ); - objectSidewaysDirection.setLength( _moveCurr.x - _movePrev.x ); - - moveDirection.copy( objectUpDirection.add( objectSidewaysDirection ) ); - - axis.crossVectors( moveDirection, _eye ).normalize(); - - angle *= _this.rotateSpeed; - quaternion.setFromAxisAngle( axis, angle ); - - _eye.applyQuaternion( quaternion ); - _this.object.up.applyQuaternion( quaternion ); - - _lastAxis.copy( axis ); - _lastAngle = angle; - - } else if ( ! _this.staticMoving && _lastAngle ) { - - _lastAngle *= Math.sqrt( 1.0 - _this.dynamicDampingFactor ); - _eye.copy( _this.object.position ).sub( _this.target ); - quaternion.setFromAxisAngle( _lastAxis, _lastAngle ); - _eye.applyQuaternion( quaternion ); - _this.object.up.applyQuaternion( quaternion ); - - } - - _movePrev.copy( _moveCurr ); - - }; - - }() ); - - - this.zoomCamera = function () { - - var factor; - - if ( _state === STATE.TOUCH_ZOOM_PAN ) { - - factor = _touchZoomDistanceStart / _touchZoomDistanceEnd; - _touchZoomDistanceStart = _touchZoomDistanceEnd; - _eye.multiplyScalar( factor ); - - } else { - - factor = 1.0 + ( _zoomEnd.y - _zoomStart.y ) * _this.zoomSpeed; - - if ( factor !== 1.0 && factor > 0.0 ) { - - _eye.multiplyScalar( factor ); - - if ( _this.staticMoving ) { - - _zoomStart.copy( _zoomEnd ); - - } else { - - _zoomStart.y += ( _zoomEnd.y - _zoomStart.y ) * this.dynamicDampingFactor; - - } - - } - - } - - }; - - this.panCamera = ( function() { - - var mouseChange = new THREE.Vector2(), - objectUp = new THREE.Vector3(), - pan = new THREE.Vector3(); - - return function panCamera() { - - mouseChange.copy( _panEnd ).sub( _panStart ); - - if ( mouseChange.lengthSq() ) { - - mouseChange.multiplyScalar( _eye.length() * _this.panSpeed ); - - pan.copy( _eye ).cross( _this.object.up ).setLength( mouseChange.x ); - pan.add( objectUp.copy( _this.object.up ).setLength( mouseChange.y ) ); - - _this.object.position.add( pan ); - _this.target.add( pan ); - - if ( _this.staticMoving ) { - - _panStart.copy( _panEnd ); - - } else { - - _panStart.add( mouseChange.subVectors( _panEnd, _panStart ).multiplyScalar( _this.dynamicDampingFactor ) ); - - } - - } - - }; - - }() ); - - this.checkDistances = function () { - - if ( ! _this.noZoom || ! _this.noPan ) { - - if ( _eye.lengthSq() > _this.maxDistance * _this.maxDistance ) { - - _this.object.position.addVectors( _this.target, _eye.setLength( _this.maxDistance ) ); - _zoomStart.copy( _zoomEnd ); - - } - - if ( _eye.lengthSq() < _this.minDistance * _this.minDistance ) { - - _this.object.position.addVectors( _this.target, _eye.setLength( _this.minDistance ) ); - _zoomStart.copy( _zoomEnd ); - - } - - } - - }; - - this.update = function () { - - _eye.subVectors( _this.object.position, _this.target ); - - if ( ! _this.noRotate ) { - - _this.rotateCamera(); - - } - - if ( ! _this.noZoom ) { - - _this.zoomCamera(); - - } - - if ( ! _this.noPan ) { - - _this.panCamera(); - - } - - _this.object.position.addVectors( _this.target, _eye ); - - _this.checkDistances(); - - _this.object.lookAt( _this.target ); - - if ( lastPosition.distanceToSquared( _this.object.position ) > EPS ) { - - _this.dispatchEvent( changeEvent ); - - lastPosition.copy( _this.object.position ); - - } - - }; - - this.reset = function () { - - _state = STATE.NONE; - _prevState = STATE.NONE; - - _this.target.copy( _this.target0 ); - _this.object.position.copy( _this.position0 ); - _this.object.up.copy( _this.up0 ); - - _eye.subVectors( _this.object.position, _this.target ); - - _this.object.lookAt( _this.target ); - - _this.dispatchEvent( changeEvent ); - - lastPosition.copy( _this.object.position ); - - }; - - // listeners - - function keydown( event ) { - - if ( _this.enabled === false ) return; - - _prevState = _state; - - if ( _state !== STATE.NONE ) { - - return; - - } else if ( event.keyCode === _this.keys[ STATE.ROTATE ] && ! _this.noRotate ) { - - _state = STATE.ROTATE; - - } else if ( event.keyCode === _this.keys[ STATE.ZOOM ] && ! _this.noZoom ) { - - _state = STATE.ZOOM; - - } else if ( event.keyCode === _this.keys[ STATE.PAN ] && ! _this.noPan ) { - - _state = STATE.PAN; - - } - - } - - function keyup( event ) { - - if ( _this.enabled === false ) return; - - _state = _prevState; - - } - - function mousedown( event ) { - - if ( _this.enabled === false ) return; - - if ( _state === STATE.NONE ) { - - _state = event.button; - - } - - if ( _state === STATE.ROTATE && ! _this.noRotate ) { - - _moveCurr.copy( getMouseOnCircle( event.pageX, event.pageY ) ); - _movePrev.copy( _moveCurr ); - - } else if ( _state === STATE.ZOOM && ! _this.noZoom ) { - - _zoomStart.copy( getMouseOnScreen( event.pageX, event.pageY ) ); - _zoomEnd.copy( _zoomStart ); - - } else if ( _state === STATE.PAN && ! _this.noPan ) { - - _panStart.copy( getMouseOnScreen( event.pageX, event.pageY ) ); - _panEnd.copy( _panStart ); - - } - - document.addEventListener( 'mousemove', mousemove, false ); - document.addEventListener( 'mouseup', mouseup, false ); - - _this.dispatchEvent( startEvent ); - - } - - function mousemove( event ) { - - if ( _this.enabled === false ) return; - - if ( _state === STATE.ROTATE && ! _this.noRotate ) { - - _movePrev.copy( _moveCurr ); - _moveCurr.copy( getMouseOnCircle( event.pageX, event.pageY ) ); - - } else if ( _state === STATE.ZOOM && ! _this.noZoom ) { - - _zoomEnd.copy( getMouseOnScreen( event.pageX, event.pageY ) ); - - } else if ( _state === STATE.PAN && ! _this.noPan ) { - - _panEnd.copy( getMouseOnScreen( event.pageX, event.pageY ) ); - - } - - } - - function mouseup( event ) { - - if ( _this.enabled === false ) return; - - _state = STATE.NONE; - - document.removeEventListener( 'mousemove', mousemove ); - document.removeEventListener( 'mouseup', mouseup ); - _this.dispatchEvent( endEvent ); - - } - - function mousewheel( event ) { - - if ( _this.enabled === false ) return; - - var delta = 0; - - if ( event.wheelDelta ) { - - // WebKit / Opera / Explorer 9 - - delta = event.wheelDelta / 40; - - } else if ( event.detail ) { - - // Firefox - - delta = - event.detail / 3; - - } - - _zoomStart.y += delta * 0.01; - _this.dispatchEvent( startEvent ); - _this.dispatchEvent( endEvent ); - - } - - function touchstart( event ) { - - if ( _this.enabled === false ) return; - - switch ( event.touches.length ) { - - case 1: - _state = STATE.TOUCH_ROTATE; - _moveCurr.copy( getMouseOnCircle( event.touches[ 0 ].pageX, event.touches[ 0 ].pageY ) ); - _movePrev.copy( _moveCurr ); - break; - - default: // 2 or more - _state = STATE.TOUCH_ZOOM_PAN; - var dx = event.touches[ 0 ].pageX - event.touches[ 1 ].pageX; - var dy = event.touches[ 0 ].pageY - event.touches[ 1 ].pageY; - _touchZoomDistanceEnd = _touchZoomDistanceStart = Math.sqrt( dx * dx + dy * dy ); - - var x = ( event.touches[ 0 ].pageX + event.touches[ 1 ].pageX ) / 2; - var y = ( event.touches[ 0 ].pageY + event.touches[ 1 ].pageY ) / 2; - _panStart.copy( getMouseOnScreen( x, y ) ); - _panEnd.copy( _panStart ); - break; - - } - - _this.dispatchEvent( startEvent ); - - } - - function touchmove( event ) { - - if ( _this.enabled === false ) return; - - switch ( event.touches.length ) { - - case 1: - _movePrev.copy( _moveCurr ); - _moveCurr.copy( getMouseOnCircle( event.touches[ 0 ].pageX, event.touches[ 0 ].pageY ) ); - break; - - default: // 2 or more - var dx = event.touches[ 0 ].pageX - event.touches[ 1 ].pageX; - var dy = event.touches[ 0 ].pageY - event.touches[ 1 ].pageY; - _touchZoomDistanceEnd = Math.sqrt( dx * dx + dy * dy ); - - var x = ( event.touches[ 0 ].pageX + event.touches[ 1 ].pageX ) / 2; - var y = ( event.touches[ 0 ].pageY + event.touches[ 1 ].pageY ) / 2; - _panEnd.copy( getMouseOnScreen( x, y ) ); - break; - - } - - } - - function touchend( event ) { - - if ( _this.enabled === false ) return; - - switch ( event.touches.length ) { - - case 0: - _state = STATE.NONE; - break; - - case 1: - _state = STATE.TOUCH_ROTATE; - _moveCurr.copy( getMouseOnCircle( event.touches[ 0 ].pageX, event.touches[ 0 ].pageY ) ); - _movePrev.copy( _moveCurr ); - break; - - } - - _this.dispatchEvent( endEvent ); - - } - - function contextmenu( event ) { - - event.preventDefault(); - - } - - this.dispose = function() { - - this.domElement.removeEventListener( 'contextmenu', contextmenu, false ); - this.domElement.removeEventListener( 'mousedown', mousedown, false ); - this.domElement.removeEventListener( 'mousewheel', mousewheel, false ); - this.domElement.removeEventListener( 'MozMousePixelScroll', mousewheel, false ); // firefox - - this.domElement.removeEventListener( 'touchstart', touchstart, false ); - this.domElement.removeEventListener( 'touchend', touchend, false ); - this.domElement.removeEventListener( 'touchmove', touchmove, false ); - - document.removeEventListener( 'mousemove', mousemove, false ); - document.removeEventListener( 'mouseup', mouseup, false ); - - window.removeEventListener( 'keydown', keydown, false ); - window.removeEventListener( 'keyup', keyup, false ); - - }; - - this.domElement.addEventListener( 'contextmenu', contextmenu, false ); - this.domElement.addEventListener( 'mousedown', mousedown, false ); - this.domElement.addEventListener( 'mousewheel', mousewheel, false ); - this.domElement.addEventListener( 'MozMousePixelScroll', mousewheel, false ); // firefox - - this.domElement.addEventListener( 'touchstart', touchstart, false ); - this.domElement.addEventListener( 'touchend', touchend, false ); - this.domElement.addEventListener( 'touchmove', touchmove, false ); - - window.addEventListener( 'keydown', keydown, false ); - window.addEventListener( 'keyup', keyup, false ); - - this.handleResize(); - - // force an update at start - this.update(); - -}; - -THREE.TrackballControls.prototype = Object.create( THREE.EventDispatcher.prototype ); -THREE.TrackballControls.prototype.constructor = THREE.TrackballControls; - diff -r bbf39c61e3fe -r e7bc59b9ebda media/js/jquery.js --- a/media/js/jquery.js Tue Sep 20 15:23:05 2016 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,151 +0,0 @@ -/*! - * jQuery JavaScript Library v1.4 - * http://jquery.com/ - * - * Copyright 2010, John Resig - * Dual licensed under the MIT or GPL Version 2 licenses. - * http://docs.jquery.com/License - * - * Includes Sizzle.js - * http://sizzlejs.com/ - * Copyright 2010, The Dojo Foundation - * Released under the MIT, BSD, and GPL Licenses. - * - * Date: Wed Jan 13 15:23:05 2010 -0500 - */ -(function(A,w){function oa(){if(!c.isReady){try{s.documentElement.doScroll("left")}catch(a){setTimeout(oa,1);return}c.ready()}}function La(a,b){b.src?c.ajax({url:b.src,async:false,dataType:"script"}):c.globalEval(b.text||b.textContent||b.innerHTML||"");b.parentNode&&b.parentNode.removeChild(b)}function $(a,b,d,f,e,i){var j=a.length;if(typeof b==="object"){for(var o in b)$(a,o,b[o],f,e,d);return a}if(d!==w){f=!i&&f&&c.isFunction(d);for(o=0;o-1){i=j.data;i.beforeFilter&&i.beforeFilter[a.type]&&!i.beforeFilter[a.type](a)||f.push(j.selector)}else delete t[p]}i=c(a.target).closest(f,a.currentTarget); -n=0;for(l=i.length;n)[^>]*$|^#([\w-]+)$/,Pa=/^.[^:#\[\.,]*$/,Qa=/\S/, -Ra=/^(\s|\u00A0)+|(\s|\u00A0)+$/g,Sa=/^<(\w+)\s*\/?>(?:<\/\1>)?$/,P=navigator.userAgent,xa=false,Q=[],M,ca=Object.prototype.toString,da=Object.prototype.hasOwnProperty,ea=Array.prototype.push,R=Array.prototype.slice,V=Array.prototype.indexOf;c.fn=c.prototype={init:function(a,b){var d,f;if(!a)return this;if(a.nodeType){this.context=this[0]=a;this.length=1;return this}if(typeof a==="string")if((d=Oa.exec(a))&&(d[1]||!b))if(d[1]){f=b?b.ownerDocument||b:s;if(a=Sa.exec(a))if(c.isPlainObject(b)){a=[s.createElement(a[1])]; -c.fn.attr.call(a,b,true)}else a=[f.createElement(a[1])];else{a=ua([d[1]],[f]);a=(a.cacheable?a.fragment.cloneNode(true):a.fragment).childNodes}}else{if(b=s.getElementById(d[2])){if(b.id!==d[2])return U.find(a);this.length=1;this[0]=b}this.context=s;this.selector=a;return this}else if(!b&&/^\w+$/.test(a)){this.selector=a;this.context=s;a=s.getElementsByTagName(a)}else return!b||b.jquery?(b||U).find(a):c(b).find(a);else if(c.isFunction(a))return U.ready(a);if(a.selector!==w){this.selector=a.selector; -this.context=a.context}return c.isArray(a)?this.setArray(a):c.makeArray(a,this)},selector:"",jquery:"1.4",length:0,size:function(){return this.length},toArray:function(){return R.call(this,0)},get:function(a){return a==null?this.toArray():a<0?this.slice(a)[0]:this[a]},pushStack:function(a,b,d){a=c(a||null);a.prevObject=this;a.context=this.context;if(b==="find")a.selector=this.selector+(this.selector?" ":"")+d;else if(b)a.selector=this.selector+"."+b+"("+d+")";return a},setArray:function(a){this.length= -0;ea.apply(this,a);return this},each:function(a,b){return c.each(this,a,b)},ready:function(a){c.bindReady();if(c.isReady)a.call(s,c);else Q&&Q.push(a);return this},eq:function(a){return a===-1?this.slice(a):this.slice(a,+a+1)},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},slice:function(){return this.pushStack(R.apply(this,arguments),"slice",R.call(arguments).join(","))},map:function(a){return this.pushStack(c.map(this,function(b,d){return a.call(b,d,b)}))},end:function(){return this.prevObject|| -c(null)},push:ea,sort:[].sort,splice:[].splice};c.fn.init.prototype=c.fn;c.extend=c.fn.extend=function(){var a=arguments[0]||{},b=1,d=arguments.length,f=false,e,i,j,o;if(typeof a==="boolean"){f=a;a=arguments[1]||{};b=2}if(typeof a!=="object"&&!c.isFunction(a))a={};if(d===b){a=this;--b}for(;b
    a";var e=d.getElementsByTagName("*"),i=d.getElementsByTagName("a")[0];if(!(!e||!e.length||!i)){c.support={leadingWhitespace:d.firstChild.nodeType===3,tbody:!d.getElementsByTagName("tbody").length, -htmlSerialize:!!d.getElementsByTagName("link").length,style:/red/.test(i.getAttribute("style")),hrefNormalized:i.getAttribute("href")==="/a",opacity:/^0.55$/.test(i.style.opacity),cssFloat:!!i.style.cssFloat,checkOn:d.getElementsByTagName("input")[0].value==="on",optSelected:s.createElement("select").appendChild(s.createElement("option")).selected,scriptEval:false,noCloneEvent:true,boxModel:null};b.type="text/javascript";try{b.appendChild(s.createTextNode("window."+f+"=1;"))}catch(j){}a.insertBefore(b, -a.firstChild);if(A[f]){c.support.scriptEval=true;delete A[f]}a.removeChild(b);if(d.attachEvent&&d.fireEvent){d.attachEvent("onclick",function o(){c.support.noCloneEvent=false;d.detachEvent("onclick",o)});d.cloneNode(true).fireEvent("onclick")}c(function(){var o=s.createElement("div");o.style.width=o.style.paddingLeft="1px";s.body.appendChild(o);c.boxModel=c.support.boxModel=o.offsetWidth===2;s.body.removeChild(o).style.display="none"});a=function(o){var p=s.createElement("div");o="on"+o;var n=o in -p;if(!n){p.setAttribute(o,"return;");n=typeof p[o]==="function"}return n};c.support.submitBubbles=a("submit");c.support.changeBubbles=a("change");a=b=d=e=i=null}})();c.props={"for":"htmlFor","class":"className",readonly:"readOnly",maxlength:"maxLength",cellspacing:"cellSpacing",rowspan:"rowSpan",colspan:"colSpan",tabindex:"tabIndex",usemap:"useMap",frameborder:"frameBorder"};var H="jQuery"+K(),Ta=0,ya={},Ua={};c.extend({cache:{},expando:H,noData:{embed:true,object:true,applet:true},data:function(a, -b,d){if(!(a.nodeName&&c.noData[a.nodeName.toLowerCase()])){a=a==A?ya:a;var f=a[H],e=c.cache;if(!b&&!f)return null;f||(f=++Ta);if(typeof b==="object"){a[H]=f;e=e[f]=c.extend(true,{},b)}else e=e[f]?e[f]:typeof d==="undefined"?Ua:(e[f]={});if(d!==w){a[H]=f;e[b]=d}return typeof b==="string"?e[b]:e}},removeData:function(a,b){if(!(a.nodeName&&c.noData[a.nodeName.toLowerCase()])){a=a==A?ya:a;var d=a[H],f=c.cache,e=f[d];if(b){if(e){delete e[b];c.isEmptyObject(e)&&c.removeData(a)}}else{try{delete a[H]}catch(i){a.removeAttribute&& -a.removeAttribute(H)}delete f[d]}}}});c.fn.extend({data:function(a,b){if(typeof a==="undefined"&&this.length)return c.data(this[0]);else if(typeof a==="object")return this.each(function(){c.data(this,a)});var d=a.split(".");d[1]=d[1]?"."+d[1]:"";if(b===w){var f=this.triggerHandler("getData"+d[1]+"!",[d[0]]);if(f===w&&this.length)f=c.data(this[0],a);return f===w&&d[1]?this.data(d[0]):f}else return this.trigger("setData"+d[1]+"!",[d[0],b]).each(function(){c.data(this,a,b)})},removeData:function(a){return this.each(function(){c.removeData(this, -a)})}});c.extend({queue:function(a,b,d){if(a){b=(b||"fx")+"queue";var f=c.data(a,b);if(!d)return f||[];if(!f||c.isArray(d))f=c.data(a,b,c.makeArray(d));else f.push(d);return f}},dequeue:function(a,b){b=b||"fx";var d=c.queue(a,b),f=d.shift();if(f==="inprogress")f=d.shift();if(f){b==="fx"&&d.unshift("inprogress");f.call(a,function(){c.dequeue(a,b)})}}});c.fn.extend({queue:function(a,b){if(typeof a!=="string"){b=a;a="fx"}if(b===w)return c.queue(this[0],a);return this.each(function(){var d=c.queue(this, -a,b);a==="fx"&&d[0]!=="inprogress"&&c.dequeue(this,a)})},dequeue:function(a){return this.each(function(){c.dequeue(this,a)})},delay:function(a,b){a=c.fx?c.fx.speeds[a]||a:a;b=b||"fx";return this.queue(b,function(){var d=this;setTimeout(function(){c.dequeue(d,b)},a)})},clearQueue:function(a){return this.queue(a||"fx",[])}});var za=/[\n\t]/g,fa=/\s+/,Va=/\r/g,Wa=/href|src|style/,Xa=/(button|input)/i,Ya=/(button|input|object|select|textarea)/i,Za=/^(a|area)$/i,Aa=/radio|checkbox/;c.fn.extend({attr:function(a, -b){return $(this,a,b,true,c.attr)},removeAttr:function(a){return this.each(function(){c.attr(this,a,"");this.nodeType===1&&this.removeAttribute(a)})},addClass:function(a){if(c.isFunction(a))return this.each(function(p){var n=c(this);n.addClass(a.call(this,p,n.attr("class")))});if(a&&typeof a==="string")for(var b=(a||"").split(fa),d=0,f=this.length;d-1)return true;return false},val:function(a){if(a===w){var b=this[0];if(b){if(c.nodeName(b,"option"))return(b.attributes.value||{}).specified?b.value:b.text;if(c.nodeName(b,"select")){var d=b.selectedIndex,f=[],e=b.options;b=b.type==="select-one";if(d<0)return null;var i=b?d:0;for(d=b?d+1:e.length;i=0;else if(c.nodeName(this,"select")){var z=c.makeArray(t);c("option",this).each(function(){this.selected=c.inArray(c(this).val(),z)>=0});if(!z.length)this.selectedIndex= --1}else this.value=t}})}});c.extend({attrFn:{val:true,css:true,html:true,text:true,data:true,width:true,height:true,offset:true},attr:function(a,b,d,f){if(!a||a.nodeType===3||a.nodeType===8)return w;if(f&&b in c.attrFn)return c(a)[b](d);f=a.nodeType!==1||!c.isXMLDoc(a);var e=d!==w;b=f&&c.props[b]||b;if(a.nodeType===1){var i=Wa.test(b);if(b in a&&f&&!i){if(e){if(b==="type"&&Xa.test(a.nodeName)&&a.parentNode)throw"type property can't be changed";a[b]=d}if(c.nodeName(a,"form")&&a.getAttributeNode(b))return a.getAttributeNode(b).nodeValue; -if(b==="tabIndex")return(b=a.getAttributeNode("tabIndex"))&&b.specified?b.value:Ya.test(a.nodeName)||Za.test(a.nodeName)&&a.href?0:w;return a[b]}if(!c.support.style&&f&&b==="style"){if(e)a.style.cssText=""+d;return a.style.cssText}e&&a.setAttribute(b,""+d);a=!c.support.hrefNormalized&&f&&i?a.getAttribute(b,2):a.getAttribute(b);return a===null?w:a}return c.style(a,b,d)}});var $a=function(a){return a.replace(/[^\w\s\.\|`]/g,function(b){return"\\"+b})};c.event={add:function(a,b,d,f){if(!(a.nodeType=== -3||a.nodeType===8)){if(a.setInterval&&a!==A&&!a.frameElement)a=A;if(!d.guid)d.guid=c.guid++;if(f!==w){d=c.proxy(d);d.data=f}var e=c.data(a,"events")||c.data(a,"events",{}),i=c.data(a,"handle"),j;if(!i){j=function(){return typeof c!=="undefined"&&!c.event.triggered?c.event.handle.apply(j.elem,arguments):w};i=c.data(a,"handle",j)}if(i){i.elem=a;b=b.split(/\s+/);for(var o,p=0;o=b[p++];){var n=o.split(".");o=n.shift();d.type=n.slice(0).sort().join(".");var t=e[o],z=this.special[o]||{};if(!t){t=e[o]={}; -if(!z.setup||z.setup.call(a,f,n,d)===false)if(a.addEventListener)a.addEventListener(o,i,false);else a.attachEvent&&a.attachEvent("on"+o,i)}if(z.add)if((n=z.add.call(a,d,f,n,t))&&c.isFunction(n)){n.guid=n.guid||d.guid;d=n}t[d.guid]=d;this.global[o]=true}a=null}}},global:{},remove:function(a,b,d){if(!(a.nodeType===3||a.nodeType===8)){var f=c.data(a,"events"),e,i,j;if(f){if(b===w||typeof b==="string"&&b.charAt(0)===".")for(i in f)this.remove(a,i+(b||""));else{if(b.type){d=b.handler;b=b.type}b=b.split(/\s+/); -for(var o=0;i=b[o++];){var p=i.split(".");i=p.shift();var n=!p.length,t=c.map(p.slice(0).sort(),$a);t=new RegExp("(^|\\.)"+t.join("\\.(?:.*\\.)?")+"(\\.|$)");var z=this.special[i]||{};if(f[i]){if(d){j=f[i][d.guid];delete f[i][d.guid]}else for(var B in f[i])if(n||t.test(f[i][B].type))delete f[i][B];z.remove&&z.remove.call(a,p,j);for(e in f[i])break;if(!e){if(!z.teardown||z.teardown.call(a,p)===false)if(a.removeEventListener)a.removeEventListener(i,c.data(a,"handle"),false);else a.detachEvent&&a.detachEvent("on"+ -i,c.data(a,"handle"));e=null;delete f[i]}}}}for(e in f)break;if(!e){if(B=c.data(a,"handle"))B.elem=null;c.removeData(a,"events");c.removeData(a,"handle")}}}},trigger:function(a,b,d,f){var e=a.type||a;if(!f){a=typeof a==="object"?a[H]?a:c.extend(c.Event(e),a):c.Event(e);if(e.indexOf("!")>=0){a.type=e=e.slice(0,-1);a.exclusive=true}if(!d){a.stopPropagation();this.global[e]&&c.each(c.cache,function(){this.events&&this.events[e]&&c.event.trigger(a,b,this.handle.elem)})}if(!d||d.nodeType===3||d.nodeType=== -8)return w;a.result=w;a.target=d;b=c.makeArray(b);b.unshift(a)}a.currentTarget=d;var i=c.data(d,"handle");i&&i.apply(d,b);var j,o;try{if(!(d&&d.nodeName&&c.noData[d.nodeName.toLowerCase()])){j=d[e];o=d["on"+e]}}catch(p){}i=c.nodeName(d,"a")&&e==="click";if(!f&&j&&!a.isDefaultPrevented()&&!i){this.triggered=true;try{d[e]()}catch(n){}}else if(o&&d["on"+e].apply(d,b)===false)a.result=false;this.triggered=false;if(!a.isPropagationStopped())(d=d.parentNode||d.ownerDocument)&&c.event.trigger(a,b,d,true)}, -handle:function(a){var b,d;a=arguments[0]=c.event.fix(a||A.event);a.currentTarget=this;d=a.type.split(".");a.type=d.shift();b=!d.length&&!a.exclusive;var f=new RegExp("(^|\\.)"+d.slice(0).sort().join("\\.(?:.*\\.)?")+"(\\.|$)");d=(c.data(this,"events")||{})[a.type];for(var e in d){var i=d[e];if(b||f.test(i.type)){a.handler=i;a.data=i.data;i=i.apply(this,arguments);if(i!==w){a.result=i;if(i===false){a.preventDefault();a.stopPropagation()}}if(a.isImmediatePropagationStopped())break}}return a.result}, -props:"altKey attrChange attrName bubbles button cancelable charCode clientX clientY ctrlKey currentTarget data detail eventPhase fromElement handler keyCode layerX layerY metaKey newValue offsetX offsetY originalTarget pageX pageY prevValue relatedNode relatedTarget screenX screenY shiftKey srcElement target toElement view wheelDelta which".split(" "),fix:function(a){if(a[H])return a;var b=a;a=c.Event(b);for(var d=this.props.length,f;d;){f=this.props[--d];a[f]=b[f]}if(!a.target)a.target=a.srcElement|| -s;if(a.target.nodeType===3)a.target=a.target.parentNode;if(!a.relatedTarget&&a.fromElement)a.relatedTarget=a.fromElement===a.target?a.toElement:a.fromElement;if(a.pageX==null&&a.clientX!=null){b=s.documentElement;d=s.body;a.pageX=a.clientX+(b&&b.scrollLeft||d&&d.scrollLeft||0)-(b&&b.clientLeft||d&&d.clientLeft||0);a.pageY=a.clientY+(b&&b.scrollTop||d&&d.scrollTop||0)-(b&&b.clientTop||d&&d.clientTop||0)}if(!a.which&&(a.charCode||a.charCode===0?a.charCode:a.keyCode))a.which=a.charCode||a.keyCode;if(!a.metaKey&& -a.ctrlKey)a.metaKey=a.ctrlKey;if(!a.which&&a.button!==w)a.which=a.button&1?1:a.button&2?3:a.button&4?2:0;return a},guid:1E8,proxy:c.proxy,special:{ready:{setup:c.bindReady,teardown:c.noop},live:{add:function(a,b){c.extend(a,b||{});a.guid+=b.selector+b.live;c.event.add(this,b.live,qa,b)},remove:function(a){if(a.length){var b=0,d=new RegExp("(^|\\.)"+a[0]+"(\\.|$)");c.each(c.data(this,"events").live||{},function(){d.test(this.type)&&b++});b<1&&c.event.remove(this,a[0],qa)}},special:{}},beforeunload:{setup:function(a, -b,d){if(this.setInterval)this.onbeforeunload=d;return false},teardown:function(a,b){if(this.onbeforeunload===b)this.onbeforeunload=null}}}};c.Event=function(a){if(!this.preventDefault)return new c.Event(a);if(a&&a.type){this.originalEvent=a;this.type=a.type}else this.type=a;this.timeStamp=K();this[H]=true};c.Event.prototype={preventDefault:function(){this.isDefaultPrevented=ba;var a=this.originalEvent;if(a){a.preventDefault&&a.preventDefault();a.returnValue=false}},stopPropagation:function(){this.isPropagationStopped= -ba;var a=this.originalEvent;if(a){a.stopPropagation&&a.stopPropagation();a.cancelBubble=true}},stopImmediatePropagation:function(){this.isImmediatePropagationStopped=ba;this.stopPropagation()},isDefaultPrevented:aa,isPropagationStopped:aa,isImmediatePropagationStopped:aa};var Ba=function(a){for(var b=a.relatedTarget;b&&b!==this;)try{b=b.parentNode}catch(d){break}if(b!==this){a.type=a.data;c.event.handle.apply(this,arguments)}},Ca=function(a){a.type=a.data;c.event.handle.apply(this,arguments)};c.each({mouseenter:"mouseover", -mouseleave:"mouseout"},function(a,b){c.event.special[a]={setup:function(d){c.event.add(this,b,d&&d.selector?Ca:Ba,a)},teardown:function(d){c.event.remove(this,b,d&&d.selector?Ca:Ba)}}});if(!c.support.submitBubbles)c.event.special.submit={setup:function(a,b,d){if(this.nodeName.toLowerCase()!=="form"){c.event.add(this,"click.specialSubmit."+d.guid,function(f){var e=f.target,i=e.type;if((i==="submit"||i==="image")&&c(e).closest("form").length)return pa("submit",this,arguments)});c.event.add(this,"keypress.specialSubmit."+ -d.guid,function(f){var e=f.target,i=e.type;if((i==="text"||i==="password")&&c(e).closest("form").length&&f.keyCode===13)return pa("submit",this,arguments)})}else return false},remove:function(a,b){c.event.remove(this,"click.specialSubmit"+(b?"."+b.guid:""));c.event.remove(this,"keypress.specialSubmit"+(b?"."+b.guid:""))}};if(!c.support.changeBubbles){var ga=/textarea|input|select/i;function Da(a){var b=a.type,d=a.value;if(b==="radio"||b==="checkbox")d=a.checked;else if(b==="select-multiple")d=a.selectedIndex> --1?c.map(a.options,function(f){return f.selected}).join("-"):"";else if(a.nodeName.toLowerCase()==="select")d=a.selectedIndex;return d}function ha(a,b){var d=a.target,f,e;if(!(!ga.test(d.nodeName)||d.readOnly)){f=c.data(d,"_change_data");e=Da(d);if(e!==f){if(a.type!=="focusout"||d.type!=="radio")c.data(d,"_change_data",e);if(d.type!=="select"&&(f!=null||e)){a.type="change";return c.event.trigger(a,b,this)}}}}c.event.special.change={filters:{focusout:ha,click:function(a){var b=a.target,d=b.type;if(d=== -"radio"||d==="checkbox"||b.nodeName.toLowerCase()==="select")return ha.call(this,a)},keydown:function(a){var b=a.target,d=b.type;if(a.keyCode===13&&b.nodeName.toLowerCase()!=="textarea"||a.keyCode===32&&(d==="checkbox"||d==="radio")||d==="select-multiple")return ha.call(this,a)},beforeactivate:function(a){a=a.target;a.nodeName.toLowerCase()==="input"&&a.type==="radio"&&c.data(a,"_change_data",Da(a))}},setup:function(a,b,d){for(var f in W)c.event.add(this,f+".specialChange."+d.guid,W[f]);return ga.test(this.nodeName)}, -remove:function(a,b){for(var d in W)c.event.remove(this,d+".specialChange"+(b?"."+b.guid:""),W[d]);return ga.test(this.nodeName)}};var W=c.event.special.change.filters}s.addEventListener&&c.each({focus:"focusin",blur:"focusout"},function(a,b){function d(f){f=c.event.fix(f);f.type=b;return c.event.handle.call(this,f)}c.event.special[b]={setup:function(){this.addEventListener(a,d,true)},teardown:function(){this.removeEventListener(a,d,true)}}});c.each(["bind","one"],function(a,b){c.fn[b]=function(d, -f,e){if(typeof d==="object"){for(var i in d)this[b](i,f,d[i],e);return this}if(c.isFunction(f)){thisObject=e;e=f;f=w}var j=b==="one"?c.proxy(e,function(o){c(this).unbind(o,j);return e.apply(this,arguments)}):e;return d==="unload"&&b!=="one"?this.one(d,f,e,thisObject):this.each(function(){c.event.add(this,d,j,f)})}});c.fn.extend({unbind:function(a,b){if(typeof a==="object"&&!a.preventDefault){for(var d in a)this.unbind(d,a[d]);return this}return this.each(function(){c.event.remove(this,a,b)})},trigger:function(a, -b){return this.each(function(){c.event.trigger(a,b,this)})},triggerHandler:function(a,b){if(this[0]){a=c.Event(a);a.preventDefault();a.stopPropagation();c.event.trigger(a,b,this[0]);return a.result}},toggle:function(a){for(var b=arguments,d=1;d0){y=u;break}}u=u[g]}m[r]=y}}}var f=/((?:\((?:\([^()]+\)|[^()]+)+\)|\[(?:\[[^[\]]*\]|['"][^'"]*['"]|[^[\]'"]+)+\]|\\.|[^ >+~,(\[\\]+)+|[>+~])(\s*,\s*)?((?:.|\r|\n)*)/g, -e=0,i=Object.prototype.toString,j=false,o=true;[0,0].sort(function(){o=false;return 0});var p=function(g,h,k,m){k=k||[];var r=h=h||s;if(h.nodeType!==1&&h.nodeType!==9)return[];if(!g||typeof g!=="string")return k;for(var q=[],v,u,y,S,I=true,N=x(h),J=g;(f.exec(""),v=f.exec(J))!==null;){J=v[3];q.push(v[1]);if(v[2]){S=v[3];break}}if(q.length>1&&t.exec(g))if(q.length===2&&n.relative[q[0]])u=ia(q[0]+q[1],h);else for(u=n.relative[q[0]]?[h]:p(q.shift(),h);q.length;){g=q.shift();if(n.relative[g])g+=q.shift(); -u=ia(g,u)}else{if(!m&&q.length>1&&h.nodeType===9&&!N&&n.match.ID.test(q[0])&&!n.match.ID.test(q[q.length-1])){v=p.find(q.shift(),h,N);h=v.expr?p.filter(v.expr,v.set)[0]:v.set[0]}if(h){v=m?{expr:q.pop(),set:B(m)}:p.find(q.pop(),q.length===1&&(q[0]==="~"||q[0]==="+")&&h.parentNode?h.parentNode:h,N);u=v.expr?p.filter(v.expr,v.set):v.set;if(q.length>0)y=B(u);else I=false;for(;q.length;){var E=q.pop();v=E;if(n.relative[E])v=q.pop();else E="";if(v==null)v=h;n.relative[E](y,v,N)}}else y=[]}y||(y=u);if(!y)throw"Syntax error, unrecognized expression: "+ -(E||g);if(i.call(y)==="[object Array]")if(I)if(h&&h.nodeType===1)for(g=0;y[g]!=null;g++){if(y[g]&&(y[g]===true||y[g].nodeType===1&&F(h,y[g])))k.push(u[g])}else for(g=0;y[g]!=null;g++)y[g]&&y[g].nodeType===1&&k.push(u[g]);else k.push.apply(k,y);else B(y,k);if(S){p(S,r,k,m);p.uniqueSort(k)}return k};p.uniqueSort=function(g){if(D){j=o;g.sort(D);if(j)for(var h=1;h":function(g,h){var k=typeof h==="string";if(k&&!/\W/.test(h)){h=h.toLowerCase();for(var m=0,r=g.length;m=0))k||m.push(v);else if(k)h[q]=false;return false},ID:function(g){return g[1].replace(/\\/g,"")},TAG:function(g){return g[1].toLowerCase()}, -CHILD:function(g){if(g[1]==="nth"){var h=/(-?)(\d*)n((?:\+|-)?\d*)/.exec(g[2]==="even"&&"2n"||g[2]==="odd"&&"2n+1"||!/\D/.test(g[2])&&"0n+"+g[2]||g[2]);g[2]=h[1]+(h[2]||1)-0;g[3]=h[3]-0}g[0]=e++;return g},ATTR:function(g,h,k,m,r,q){h=g[1].replace(/\\/g,"");if(!q&&n.attrMap[h])g[1]=n.attrMap[h];if(g[2]==="~=")g[4]=" "+g[4]+" ";return g},PSEUDO:function(g,h,k,m,r){if(g[1]==="not")if((f.exec(g[3])||"").length>1||/^\w/.test(g[3]))g[3]=p(g[3],null,null,h);else{g=p.filter(g[3],h,k,true^r);k||m.push.apply(m, -g);return false}else if(n.match.POS.test(g[0])||n.match.CHILD.test(g[0]))return true;return g},POS:function(g){g.unshift(true);return g}},filters:{enabled:function(g){return g.disabled===false&&g.type!=="hidden"},disabled:function(g){return g.disabled===true},checked:function(g){return g.checked===true},selected:function(g){return g.selected===true},parent:function(g){return!!g.firstChild},empty:function(g){return!g.firstChild},has:function(g,h,k){return!!p(k[3],g).length},header:function(g){return/h\d/i.test(g.nodeName)}, -text:function(g){return"text"===g.type},radio:function(g){return"radio"===g.type},checkbox:function(g){return"checkbox"===g.type},file:function(g){return"file"===g.type},password:function(g){return"password"===g.type},submit:function(g){return"submit"===g.type},image:function(g){return"image"===g.type},reset:function(g){return"reset"===g.type},button:function(g){return"button"===g.type||g.nodeName.toLowerCase()==="button"},input:function(g){return/input|select|textarea|button/i.test(g.nodeName)}}, -setFilters:{first:function(g,h){return h===0},last:function(g,h,k,m){return h===m.length-1},even:function(g,h){return h%2===0},odd:function(g,h){return h%2===1},lt:function(g,h,k){return hk[3]-0},nth:function(g,h,k){return k[3]-0===h},eq:function(g,h,k){return k[3]-0===h}},filter:{PSEUDO:function(g,h,k,m){var r=h[1],q=n.filters[r];if(q)return q(g,k,h,m);else if(r==="contains")return(g.textContent||g.innerText||a([g])||"").indexOf(h[3])>=0;else if(r==="not"){h= -h[3];k=0;for(m=h.length;k=0}},ID:function(g,h){return g.nodeType===1&&g.getAttribute("id")===h},TAG:function(g,h){return h==="*"&&g.nodeType===1||g.nodeName.toLowerCase()===h},CLASS:function(g,h){return(" "+(g.className||g.getAttribute("class"))+" ").indexOf(h)>-1},ATTR:function(g,h){var k=h[1];g=n.attrHandle[k]?n.attrHandle[k](g):g[k]!=null?g[k]:g.getAttribute(k);k=g+"";var m=h[2];h=h[4];return g==null?m==="!=":m=== -"="?k===h:m==="*="?k.indexOf(h)>=0:m==="~="?(" "+k+" ").indexOf(h)>=0:!h?k&&g!==false:m==="!="?k!==h:m==="^="?k.indexOf(h)===0:m==="$="?k.substr(k.length-h.length)===h:m==="|="?k===h||k.substr(0,h.length+1)===h+"-":false},POS:function(g,h,k,m){var r=n.setFilters[h[2]];if(r)return r(g,k,h,m)}}},t=n.match.POS;for(var z in n.match){n.match[z]=new RegExp(n.match[z].source+/(?![^\[]*\])(?![^\(]*\))/.source);n.leftMatch[z]=new RegExp(/(^(?:.|\r|\n)*?)/.source+n.match[z].source.replace(/\\(\d+)/g,function(g, -h){return"\\"+(h-0+1)}))}var B=function(g,h){g=Array.prototype.slice.call(g,0);if(h){h.push.apply(h,g);return h}return g};try{Array.prototype.slice.call(s.documentElement.childNodes,0)}catch(C){B=function(g,h){h=h||[];if(i.call(g)==="[object Array]")Array.prototype.push.apply(h,g);else if(typeof g.length==="number")for(var k=0,m=g.length;k";var k=s.documentElement;k.insertBefore(g,k.firstChild);if(s.getElementById(h)){n.find.ID=function(m,r,q){if(typeof r.getElementById!=="undefined"&&!q)return(r=r.getElementById(m[1]))?r.id===m[1]||typeof r.getAttributeNode!=="undefined"&& -r.getAttributeNode("id").nodeValue===m[1]?[r]:w:[]};n.filter.ID=function(m,r){var q=typeof m.getAttributeNode!=="undefined"&&m.getAttributeNode("id");return m.nodeType===1&&q&&q.nodeValue===r}}k.removeChild(g);k=g=null})();(function(){var g=s.createElement("div");g.appendChild(s.createComment(""));if(g.getElementsByTagName("*").length>0)n.find.TAG=function(h,k){k=k.getElementsByTagName(h[1]);if(h[1]==="*"){h=[];for(var m=0;k[m];m++)k[m].nodeType===1&&h.push(k[m]);k=h}return k};g.innerHTML=""; -if(g.firstChild&&typeof g.firstChild.getAttribute!=="undefined"&&g.firstChild.getAttribute("href")!=="#")n.attrHandle.href=function(h){return h.getAttribute("href",2)};g=null})();s.querySelectorAll&&function(){var g=p,h=s.createElement("div");h.innerHTML="

    ";if(!(h.querySelectorAll&&h.querySelectorAll(".TEST").length===0)){p=function(m,r,q,v){r=r||s;if(!v&&r.nodeType===9&&!x(r))try{return B(r.querySelectorAll(m),q)}catch(u){}return g(m,r,q,v)};for(var k in g)p[k]=g[k];h=null}}(); -(function(){var g=s.createElement("div");g.innerHTML="
    ";if(!(!g.getElementsByClassName||g.getElementsByClassName("e").length===0)){g.lastChild.className="e";if(g.getElementsByClassName("e").length!==1){n.order.splice(1,0,"CLASS");n.find.CLASS=function(h,k,m){if(typeof k.getElementsByClassName!=="undefined"&&!m)return k.getElementsByClassName(h[1])};g=null}}})();var F=s.compareDocumentPosition?function(g,h){return g.compareDocumentPosition(h)&16}:function(g, -h){return g!==h&&(g.contains?g.contains(h):true)},x=function(g){return(g=(g?g.ownerDocument||g:0).documentElement)?g.nodeName!=="HTML":false},ia=function(g,h){var k=[],m="",r;for(h=h.nodeType?[h]:h;r=n.match.PSEUDO.exec(g);){m+=r[0];g=g.replace(n.match.PSEUDO,"")}g=n.relative[g]?g+"*":g;r=0;for(var q=h.length;r=0===d})};c.fn.extend({find:function(a){for(var b=this.pushStack("","find",a),d=0,f=0,e=this.length;f0)for(var i=d;i0},closest:function(a,b){if(c.isArray(a)){var d=[],f=this[0],e,i= -{},j;if(f&&a.length){e=0;for(var o=a.length;e-1:c(f).is(e)){d.push({selector:j,elem:f});delete i[j]}}f=f.parentNode}}return d}var p=c.expr.match.POS.test(a)?c(a,b||this.context):null;return this.map(function(n,t){for(;t&&t.ownerDocument&&t!==b;){if(p?p.index(t)>-1:c(t).is(a))return t;t=t.parentNode}return null})},index:function(a){if(!a||typeof a=== -"string")return c.inArray(this[0],a?c(a):this.parent().children());return c.inArray(a.jquery?a[0]:a,this)},add:function(a,b){a=typeof a==="string"?c(a,b||this.context):c.makeArray(a);b=c.merge(this.get(),a);return this.pushStack(sa(a[0])||sa(b[0])?b:c.unique(b))},andSelf:function(){return this.add(this.prevObject)}});c.each({parent:function(a){return(a=a.parentNode)&&a.nodeType!==11?a:null},parents:function(a){return c.dir(a,"parentNode")},parentsUntil:function(a,b,d){return c.dir(a,"parentNode", -d)},next:function(a){return c.nth(a,2,"nextSibling")},prev:function(a){return c.nth(a,2,"previousSibling")},nextAll:function(a){return c.dir(a,"nextSibling")},prevAll:function(a){return c.dir(a,"previousSibling")},nextUntil:function(a,b,d){return c.dir(a,"nextSibling",d)},prevUntil:function(a,b,d){return c.dir(a,"previousSibling",d)},siblings:function(a){return c.sibling(a.parentNode.firstChild,a)},children:function(a){return c.sibling(a.firstChild)},contents:function(a){return c.nodeName(a,"iframe")? -a.contentDocument||a.contentWindow.document:c.makeArray(a.childNodes)}},function(a,b){c.fn[a]=function(d,f){var e=c.map(this,b,d);ab.test(a)||(f=d);if(f&&typeof f==="string")e=c.filter(f,e);e=this.length>1?c.unique(e):e;if((this.length>1||cb.test(f))&&bb.test(a))e=e.reverse();return this.pushStack(e,a,R.call(arguments).join(","))}});c.extend({filter:function(a,b,d){if(d)a=":not("+a+")";return c.find.matches(a,b)},dir:function(a,b,d){var f=[];for(a=a[b];a&&a.nodeType!==9&&(d===w||!c(a).is(d));){a.nodeType=== -1&&f.push(a);a=a[b]}return f},nth:function(a,b,d){b=b||1;for(var f=0;a;a=a[d])if(a.nodeType===1&&++f===b)break;return a},sibling:function(a,b){for(var d=[];a;a=a.nextSibling)a.nodeType===1&&a!==b&&d.push(a);return d}});var Ga=/ jQuery\d+="(?:\d+|null)"/g,Y=/^\s+/,db=/(<([\w:]+)[^>]*?)\/>/g,eb=/^(?:area|br|col|embed|hr|img|input|link|meta|param)$/i,Ha=/<([\w:]+)/,fb=/"},G={option:[1,""], -legend:[1,"
    ","
    "],thead:[1,"","
    "],tr:[2,"","
    "],td:[3,"","
    "],col:[2,"","
    "],area:[1,"",""],_default:[0,"",""]};G.optgroup=G.option;G.tbody=G.tfoot=G.colgroup=G.caption=G.thead;G.th=G.td;if(!c.support.htmlSerialize)G._default=[1,"div
    ","
    "];c.fn.extend({text:function(a){if(c.isFunction(a))return this.each(function(b){var d=c(this); -return d.text(a.call(this,b,d.text()))});if(typeof a!=="object"&&a!==w)return this.empty().append((this[0]&&this[0].ownerDocument||s).createTextNode(a));return c.getText(this)},wrapAll:function(a){if(c.isFunction(a))return this.each(function(d){c(this).wrapAll(a.call(this,d))});if(this[0]){var b=c(a,this[0].ownerDocument).eq(0).clone(true);this[0].parentNode&&b.insertBefore(this[0]);b.map(function(){for(var d=this;d.firstChild&&d.firstChild.nodeType===1;)d=d.firstChild;return d}).append(this)}return this}, -wrapInner:function(a){return this.each(function(){var b=c(this),d=b.contents();d.length?d.wrapAll(a):b.append(a)})},wrap:function(a){return this.each(function(){c(this).wrapAll(a)})},unwrap:function(){return this.parent().each(function(){c.nodeName(this,"body")||c(this).replaceWith(this.childNodes)}).end()},append:function(){return this.domManip(arguments,true,function(a){this.nodeType===1&&this.appendChild(a)})},prepend:function(){return this.domManip(arguments,true,function(a){this.nodeType===1&& -this.insertBefore(a,this.firstChild)})},before:function(){if(this[0]&&this[0].parentNode)return this.domManip(arguments,false,function(b){this.parentNode.insertBefore(b,this)});else if(arguments.length){var a=c(arguments[0]);a.push.apply(a,this.toArray());return this.pushStack(a,"before",arguments)}},after:function(){if(this[0]&&this[0].parentNode)return this.domManip(arguments,false,function(b){this.parentNode.insertBefore(b,this.nextSibling)});else if(arguments.length){var a=this.pushStack(this, -"after",arguments);a.push.apply(a,c(arguments[0]).toArray());return a}},clone:function(a){var b=this.map(function(){if(!c.support.noCloneEvent&&!c.isXMLDoc(this)){var d=this.outerHTML,f=this.ownerDocument;if(!d){d=f.createElement("div");d.appendChild(this.cloneNode(true));d=d.innerHTML}return c.clean([d.replace(Ga,"").replace(Y,"")],f)[0]}else return this.cloneNode(true)});if(a===true){ta(this,b);ta(this.find("*"),b.find("*"))}return b},html:function(a){if(a===w)return this[0]&&this[0].nodeType=== -1?this[0].innerHTML.replace(Ga,""):null;else if(typeof a==="string"&&!/ + + + + + + + + + + + + + +
    +
    +
    + steve losh +
    + + +
    + +
     
    + +
    diff -r bbf39c61e3fe -r e7bc59b9ebda themes/stevelosh/theme.toml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/themes/stevelosh/theme.toml Fri Oct 07 12:48:36 2016 +0000 @@ -0,0 +1,8 @@ +name = "Stevelosh" +license = "MIT" +homepage = "http://stevelosh.com/" +min_version = 0.15 + +[author] + name = "Steve Losh" + homepage = "http://stevelosh.com/" diff -r bbf39c61e3fe -r e7bc59b9ebda yuicompressor-2.4.2.jar Binary file yuicompressor-2.4.2.jar has changed