From 4666014f583002b0204528a02e07bf6c8d247019 Mon Sep 17 00:00:00 2001 From: Micah R Ledbetter Date: Wed, 25 Jan 2017 11:52:45 -0600 Subject: [PATCH] Minor improvements to help with collaboration - Rename buildpacker to buildlab, since it's not just a packer thing - Clean up hardcoded crap and references to (the old Python version of) caryatid from buildlab - Delete code as much as possible, shortening file by ~30% - Minor readme cleanup --- buildpacker.py => buildlab.py | 103 +++++----------------------------- readme.markdown | 77 ++++++++++++++++++------- 2 files changed, 72 insertions(+), 108 deletions(-) rename buildpacker.py => buildlab.py (58%) diff --git a/buildpacker.py b/buildlab.py similarity index 58% rename from buildpacker.py rename to buildlab.py index 5e0ee89..2dc3d58 100644 --- a/buildpacker.py +++ b/buildlab.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 import argparse -import datetime import glob +import logging import os import shutil import subprocess @@ -10,46 +10,12 @@ import sys scriptdir = os.path.dirname(os.path.realpath(__file__)) -debug = False -verbose = False - - -# TODO: use logging module? - - -def strace(): - import pdb - pdb.set_trace() - - -def debugprint(message): - global debug - if debug: - print("DEBUG: " + message) - - -def verboseprint(message): - global verbose - if verbose: - print("VERBOSE: " + message) def resolvepath(path): return os.path.realpath(os.path.normpath(os.path.expanduser(path))) -def which(commandname): - def is_exe(fpath): - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - for path in os.environ["PATH"].split(os.pathsep): - path = path.strip('"') - exe_file = os.path.join(path, commandname) - if is_exe(exe_file): - return exe_file - print("Exe path is {}".format(exe_file)) - raise Exception("No such command '{}' in %PATH%".format(commandname)) - - def buildpacker(packerfile, outdir, force=False, whatif=False): packerfile = resolvepath(packerfile) if not os.path.isfile(packerfile): @@ -70,9 +36,7 @@ def buildpacker(packerfile, outdir, force=False, whatif=False): else: raise Exception("A packer output directory exists at '{}'".format(oldoutput)) - caryatidpath = resolvepath("~/Documents/caryatid") - caryatiddestination = resolvepath("D:\\Micah\\Vagrant") - cli = 'packer.exe build -var output_directory="{}" -var caryatidpath="{}" -var caryatid_destination="{}" {}'.format(outdir, caryatidpath, caryatiddestination, packerfile) + cli = 'packer.exe build -var output_directory="{}" {}'.format(outdir, packerfile) # NOTE: Packer gives a very weird error if you do not COPY the entire environment # When I was setting env to be just a dictionary with the PACKER_* variables I needed, @@ -86,11 +50,10 @@ def buildpacker(packerfile, outdir, force=False, whatif=False): env['PACKER_LOG_PATH'] = logdir env['CHECKPOINT_DISABLE'] = '1' - verboseprint("Running command:\n {}\n from directory: {}\n with environment:\n {}".format(cli, packerdir, env)) + logging.info("Running command:\n {}\n from directory: {}\n with environment:\n {}".format(cli, packerdir, env)) if whatif: return - else: - subprocess.check_call(cli, env=env, cwd=packerdir) + subprocess.check_call(cli, env=env, cwd=packerdir) boxes = glob.glob("{}/*.box".format(outdir)) if len(boxes) > 1: @@ -98,18 +61,8 @@ def buildpacker(packerfile, outdir, force=False, whatif=False): elif len(boxes) < 1: raise Exception("Apparently the packer process failed, no boxes were created") - verboseprint("Packed .box file: '{}'".format(boxes[0])) - - -def add2caryatid(name, description, version, provider, artifact, backend, destination, extension=None): - """Add a box to a JSON catalog using Caryatid - This is a temporary hack until we can get shell-local (or something similar) working on Windows - """ - import importlib.util - caryatidspec = importlib.util.spec_from_file_location("caryatid", resolvepath("~/Documents/caryatid/caryatid.py")) - caryatid = importlib.util.module_from_spec(caryatidspec) - caryatidspec.loader.exec_module(caryatid) - caryatid.addcopy(name, description, version, provider, artifact, destination) + logging.info("Packed .box file: '{}'".format(boxes[0])) + return boxes[0] def addvagrantbox(vagrantboxname, packedboxpath, force, whatif): @@ -122,7 +75,7 @@ def addvagrantbox(vagrantboxname, packedboxpath, force, whatif): forcearg = '--force' if force else '' cli = "vagrant.exe box add {} --name {} {}".format(forcearg, vagrantboxname, packedboxname) - print("Running caryatid:\n {}".format(cli)) + print("Running vagrant:\n {}".format(cli)) if whatif: return else: @@ -136,14 +89,14 @@ def main(*args, **kwargs): parser.add_argument( "baseconfigname", action='store', - help="The name of one of the subdirs of this directory, like windows_81_x86") + help="The name of one of the subdirs of the 'packer' directory, like windows_81_x86") parser.add_argument( - "--base-out-dir", "-o", action='store', default="~/Documents/WinTrialLab", + "--base-out-dir", "-o", action='store', default="{}/output".format(scriptdir), help="The base output directory, where Packer does its work and saves its final output. (NOT the VM directory, which is a setting in VirtualBox.)") parser.add_argument( - "--action", "-a", action='store', default="packercaryatid", - choices=['packer', 'vagrant', 'caryatid', 'packervagrant', 'packercaryatid'], + "--action", "-a", action='store', default="packervagrant", + choices=['packer', 'vagrant', 'packervagrant'], help="The action to perform. By default, build with packer and add to vagrant.") parser.add_argument( "--whatif", "-w", action='store_true', @@ -151,50 +104,24 @@ def main(*args, **kwargs): parser.add_argument( "--force", "-f", action='store_true', help="Force continue, even if old output directories already exist") - parser.add_argument( - "--debug", "-d", action='store_true', - help="Print debug and verbose messages") parser.add_argument( "--verbose", "-v", action='store_true', help="Print verbose messages") parsed = parser.parse_args() - if parsed.debug: - global debug - debug = True - if parsed.debug or parsed.verbose: - global verbose - verbose = True - - if parsed.action == "packervagrant": - actions = ['packer', 'vagrant'] - elif parsed.action == "packercaryatid": - actions = ['packer', 'caryatid'] - else: - actions = [parsed.action] - + if parsed.verbose: + logging.basicConfig(level=logging.DEBUG) fullconfigname = "wintriallab-{}".format(parsed.baseconfigname) packeroutdir = os.path.join(resolvepath(parsed.base_out_dir), fullconfigname) packerfile = os.path.join(scriptdir, 'packer', parsed.baseconfigname, '{}_packerfile.json'.format(parsed.baseconfigname)) - if 'packer' in actions: + if 'packer' in parsed.action: buildpacker(packerfile, packeroutdir, force=parsed.force, whatif=parsed.whatif) packedboxpath = glob.glob("{}/{}_*_virtualbox.box".format(packeroutdir, parsed.baseconfigname))[0] - if 'vagrant' in actions: + if 'vagrant' in parsed.action: addvagrantbox(fullconfigname, packedboxpath, force=parsed.force, whatif=parsed.whatif) - if 'caryatid' in actions: - now = datetime.datetime.now() - add2caryatid( - 'wintriallab-win10-32', - 'Windows Trial Lab: Windows 10 32-bit', - # '1.{}.{}'.format(now.strftime('%Y%m%d'), now.strftime('%H%M%S')), - '1.0.{}'.format(now.strftime('%Y%m%d%H%M%S')), - 'virtualbox', - packedboxpath, - 'copy', - 'E:\\Micah\\caryatid') if __name__ == '__main__': diff --git a/readme.markdown b/readme.markdown index cf12683..0f6b056 100644 --- a/readme.markdown +++ b/readme.markdown @@ -1,12 +1,55 @@ windows-trial-lab: scripts for building one or more machines from Windows trial ISOs +## Prerequisites + +- Python (for the buildlab script) +- VirtualBox +- Packer +- Vagrant + +## Using buildlab + +The buildlab script is just a wrapper script around packer.exe and vagrant.exe. It can build a packer image and import it into vagrant. + + > .\buildlab.py -h + usage: buildlab.py [-h] [--base-out-dir BASE_OUT_DIR] + [--action {packer,vagrant,packervagrant}] [--whatif] + [--force] [--verbose] + baseconfigname + + Windows Trial Lab: build trial Vagrant boxes. + + positional arguments: + baseconfigname The name of one of the subdirs of the 'packer' + directory, like windows_81_x86 + + optional arguments: + -h, --help show this help message and exit + --base-out-dir BASE_OUT_DIR, -o BASE_OUT_DIR + The base output directory, where Packer does its work + and saves its final output. (NOT the VM directory, + which is a setting in VirtualBox.) + --action {packer,vagrant,packervagrant}, -a {packer,vagrant,packervagrant} + The action to perform. By default, build with packer + and add to vagrant. + --whatif, -w Do not perform any actions, only say what would have + been done + --force, -f Force continue, even if old output directories already + exist + --verbose, -v Print verbose messages + + NOTE: requires packer 0.8.6 or higher and vagrant 1.8 or higher. EXAMPLE: + buildlab --baseconfigname windows_10_x86; cd vagrant/FreyjaA; vagrant up + +Note that doing the actual `vagrant up` is not part of buildlab - it only makes the box available for you to `vagrant up` later. See my example Vagrant boxes in the vagrant subdirectory, but note that these will be specific to my use; you'll probably want to define your own Vagrantfile(s) with your own provisioner scripts. + ## Credits -This started as some customizations for [joefitzgerald/packer-windows](https://github.com/joefitzgerald/packer-windows) that got a liiiiiiiittle out of hand. +This started as some customizations for [joefitzgerald/packer-windows](https://github.com/joefitzgerald/packer-windows) that got a liiiiiiiittle out of hand. -These were the *types* of changes I'm trying to make: +These were the *types* of changes I'm trying to make: -- I rewrote their Windows Update script to be much more readable (imo). Now it has clearly defined functions with parameter blocks, you can set the postinstall step when calling it (rather than hardcoding calling `A:\openssh.ps1`) and you only have to set it once, and all functions MAY read global variables set at the top level but DO NOT write to them. +- I rewrote their Windows Update script to be much more readable (imo). Now it has clearly defined functions with parameter blocks, you can set the postinstall step when calling it (rather than hardcoding calling `A:\openssh.ps1`) and you only have to set it once, and all functions MAY read global variables set at the top level but DO NOT write to them. - I want to use WinRM rather than OpenSSH - As a result of this, I don't copy anything to the host for provisioning, because this is buggy with WinRM. This isn't a big deal though - I just put everything I want to use on the A:\ drive and use the "attach" guest additions mode in Packer - I have a much easier time dealing with my provisioners though @@ -17,8 +60,8 @@ These were the *types* of changes I'm trying to make: And these are some specific changes that may impact you -- The original project has [a way to install KB2842230](https://github.com/joefitzgerald/packer-windows/blob/master/scripts/hotfix-KB2842230.bat). I haven't run into this problem, but if I did, I'd have to figure this one out too. I'm not sure but it appears that they have an installer script but not a downloader script - it's unclear whether people are actually using this or not. -- The original project has [a script that forces all network locations to be private](https://github.com/joefitzgerald/packer-windows/blob/master/scripts/fixnetwork.ps1), which is necessary to enable PS Remoting. I haven't hit a problem that this solved yet, so I don't include it. +- The original project has [a way to install KB2842230](https://github.com/joefitzgerald/packer-windows/blob/master/scripts/hotfix-KB2842230.bat). I haven't run into this problem, but if I did, I'd have to figure this one out too. I'm not sure but it appears that they have an installer script but not a downloader script - it's unclear whether people are actually using this or not. +- The original project has [a script that forces all network locations to be private](https://github.com/joefitzgerald/packer-windows/blob/master/scripts/fixnetwork.ps1), which is necessary to enable PS Remoting. I haven't hit a problem that this solved yet, so I don't include it. - The Windows 10 Autounattend.xml also sets the NewNetworkWindowOff registry key, per , by doing `cmd.exe /c reg add "HKLM\System\CurrentControlSet\Control\Network\NewNetworkWindowOff"`, before running the fixnetwork.ps1 script. - I don't have VMware, only VirtualBox, so all the VMware support was removed (since I was rewriting it and couldn't test it, this seemed like the right thing to do) - I use WinRM rather than OpenSSH @@ -37,29 +80,24 @@ And these are some specific changes that may impact you - provisioner-postinstall.ps1 # run by a packer provisioner, contains hardcoded values - win-updates.ps1 # run from autounattend-postinstall if desired, reboots system repeatedly - enable-winrm.ps1 # run from autounattend-postinstall - - packer/ + - packer/ - (folders for each version of Windows) ## To do -buildlab.ps1 improvements: - -- would like to use the -tag in the name for the vagrant box too, but that requires parameterizing both the packerfile and the vagrantfile template :/ not sure what to do about this -- I have a concept of "packer basename" and "tag" in buildlab. Extend this to also have "architecture" and "flavor" (or something - to capture Server Standard vs Core vs Datacenter etc) - packer/vagrant/postinstall improvements: - store passwords securely for shit and/or generate them on the fly - use client certs for WinRM: https://msdn.microsoft.com/en-us/library/aa384295%28v=vs.85%29.aspx ?? only if packer/vagrant can support it tho -- would be great if I didn't have duplicated Autounattend.xml files everywhere - can I templatize this? -- in Autounattend.xml, we turn off UAC. (That's the `false` setting.) Is this really required? Or was it only required for using shitty SSH? +- would be great if I didn't have duplicated Autounattend.xml files everywhere - can I templatize this? +- in Autounattend.xml, we turn off UAC. (That's the `false` setting.) Is this really required? Or was it only required for using shitty SSH? vagrant provisioners - decide on a systems management system. DSC seems like maybe the most natural option. -- pull down git, conemu, my dhd repo +- pull down git, conemu, my dhd repo - configure launch bar -- configure taskbar +- configure taskbar other improvements @@ -67,15 +105,14 @@ other improvements upstream improvements -- It's possible that the original project might be interested in some of the stuff I've done, particularly the Windows Update work, and maybe even my postinstall module. Clean up the code and submit it to them and see what they think. +- It's possible that the original project might be interested in some of the stuff I've done, particularly the Windows Update work, and maybe even my postinstall module. Clean up the code and submit it to them and see what they think. ## Whines - The shell, windows-shell, and powershell provisioners are VERY finicky. I canNOT make them work reliably. The easiest thing I can figure out how to do is to use a Powershell provisioner to call a file with no arguments over WinRM. lmfao - However the situation was much improved when I switched to WinRM with the powershell provisioner. That seems to work OK - I think the problem was that using the shell provisioner with OpenSSH, which provides an emulated POSIX environment of some kind -- There's lots of information on the Internet claiming you can use `HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\RunServices` (or `RunServicesOnce`) to run something at boot, before logon - analogous to the `Run`/`RunOnce` keys. This is apparently false for any NT operating system. Properties on these keys DO NOT RUN AT BOOT - they are completely ignored by the operating system. +- There's lots of information on the Internet claiming you can use `HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\RunServices` (or `RunServicesOnce`) to run something at boot, before logon - analogous to the `Run`/`RunOnce` keys. This is apparently false for any NT operating system. Properties on these keys DO NOT RUN AT BOOT - they are completely ignored by the operating system. - The original packer-windows crew got aroudn this by using the `Run` key and disabling UAC in `Autounattend.xml` - - I'm planning to get around this by creating a scheduled task that starts at boot and runs with highest privileges. This won't work pre-Vista/2008, but that's OK with me. - - This means I need to write an executor that can start at boot, and then check for things to execute located elsewhere. Bleh. - + - I'm planning to get around this by creating a scheduled task that starts at boot and runs with highest privileges. This won't work pre-Vista/2008, but that's OK with me. + - This means I need to write an executor that can start at boot, and then check for things to execute located elsewhere. Bleh.