diff --git a/Makefile b/Makefile index 68825e5..ba215fe 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -PY?=python3 +PY?=~/miniconda3/bin/python PELICAN?=pelican PELICANOPTS= diff --git a/_nb_header.html b/_nb_header.html index 4b3566d..defd29d 100644 --- a/_nb_header.html +++ b/_nb_header.html @@ -1,1545 +1,42 @@ @@ -1550,8 +47,10 @@ span#checkpoint_status,span#autosave_status{font-size:small} .highlight .err { border: 1px solid #FF0000 } /* Error */ .highlight .k { color: #008000; font-weight: bold } /* Keyword */ .highlight .o { color: #666666 } /* Operator */ +.highlight .ch { color: #408080; font-style: italic } /* Comment.Hashbang */ .highlight .cm { color: #408080; font-style: italic } /* Comment.Multiline */ .highlight .cp { color: #BC7A00 } /* Comment.Preproc */ +.highlight .cpf { color: #408080; font-style: italic } /* Comment.PreprocFile */ .highlight .c1 { color: #408080; font-style: italic } /* Comment.Single */ .highlight .cs { color: #408080; font-style: italic } /* Comment.Special */ .highlight .gd { color: #A00000 } /* Generic.Deleted */ diff --git a/ansible/library/conda.py b/ansible/library/conda.py new file mode 100644 index 0000000..469250f --- /dev/null +++ b/ansible/library/conda.py @@ -0,0 +1,278 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +DOCUMENTATION = """ +--- +module: conda +short_description: Manage Python libraries via conda +description: + > + Manage Python libraries via conda. + Can install, update, and remove packages. +author: Synthicity +notes: + > + Requires conda to already be installed. + Will look under the home directory for a conda executable. +options: + name: + description: The name of a Python library to install + required: true + default: null + version: + description: A specific version of a library to install + required: false + default: null + state: + description: State in which to leave the Python package + required: false + default: present + choices: [ "present", "absent", "latest" ] + channels: + description: Extra channels to use when installing packages + required: false + default: null + executable: + description: Full path to the conda executable + required: false + default: null + extra_args: + description: Extra arguments passed to conda + required: false + default: null +""" + +EXAMPLES = """ +- name: install numpy via conda + conda: name=numpy state=latest + +- name: install scipy 0.14 via conda + conda: name=scipy version="0.14" + +- name: remove matplotlib from conda + conda: name=matplotlib state=absent +""" + +from distutils.spawn import find_executable +import os.path + + +def _find_conda(module, executable): + """ + If `executable` is not None, checks whether it points to a valid file + and returns it if this is the case. Otherwise tries to find the `conda` + executable in the path. Calls `fail_json` if either of these fail. + + """ + if not executable: + conda = find_executable('conda') + if conda: + return conda + else: + if os.path.isfile(executable): + return executable + + module.fail_json(msg="could not find conda executable") + + +def _add_channels_to_command(command, channels): + """ + Add extra channels to a conda command by splitting the channels + and putting "--channel" before each one. + + """ + if channels: + channels = channels.strip().split() + dashc = [] + for channel in channels: + dashc.append('--channel') + dashc.append(channel) + + return command[:2] + dashc + command[2:] + else: + return command + + +def _add_extras_to_command(command, extras): + """ + Add extra arguments to a conda command by splitting the arguments + on white space and inserting them after the second item in the command. + + """ + if extras: + extras = extras.strip().split() + return command[:2] + extras + command[2:] + else: + return command + + +def _check_installed(module, conda, name): + """ + Check whether a package is installed. Returns (bool, version_str). + + """ + command = [ + conda, + 'list', + '^' + name + '$' # use regex to get an exact match + ] + command = _add_extras_to_command(command, module.params['extra_args']) + + rc, stdout, stderr = module.run_command(command) + + if rc != 0: + return False, None + + version = stdout.strip().split()[-2] + + return True, version + + +def _remove_package(module, conda, installed, name): + """ + Use conda to remove a given package if it is installed. + + """ + if module.check_mode and installed: + module.exit_json(changed=True) + + if not installed: + module.exit_json(changed=False) + + command = [ + conda, + 'remove', + '--yes', + name + ] + command = _add_extras_to_command(command, module.params['extra_args']) + + rc, stdout, stderr = module.run_command(command) + + if rc != 0: + module.fail_json(msg='failed to remove package ' + name) + + module.exit_json(changed=True, name=name, stdout=stdout, stderr=stderr) + + +def _install_package( + module, conda, installed, name, version, installed_version): + """ + Install a package at a specific version, or install a missing package at + the latest version if no version is specified. + + """ + if installed and (version is None or installed_version == version): + module.exit_json(changed=False, name=name, version=version) + + if module.check_mode: + if not installed or (installed and installed_version != version): + module.exit_json(changed=True) + + if version: + install_str = name + '=' + version + else: + install_str = name + + command = [ + conda, + 'install', + '--yes', + install_str + ] + command = _add_channels_to_command(command, module.params['channels']) + command = _add_extras_to_command(command, module.params['extra_args']) + + rc, stdout, stderr = module.run_command(command) + + if rc != 0: + module.fail_json(msg='failed to install package ' + name) + + module.exit_json( + changed=True, name=name, version=version, stdout=stdout, stderr=stderr) + + +def _update_package(module, conda, installed, name): + """ + Make sure an installed package is at its latest version. + + """ + if not installed: + module.fail_json(msg='can\'t update a package that is not installed') + + # see if it's already installed at the latest version + command = [ + conda, + 'update', + '--dry-run', + name + ] + command = _add_channels_to_command(command, module.params['channels']) + command = _add_extras_to_command(command, module.params['extra_args']) + + rc, stdout, stderr = module.run_command(command) + + if rc != 0: + module.fail_json(msg='can\'t update a package that is not installed') + + if 'requested packages already installed' in stdout: + module.exit_json(changed=False, name=name) + + # now we're definitely gonna update the package + if module.check_mode: + module.exit_json(changed=True, name=name) + + command = [ + conda, + 'update', + '--yes', + name + ] + command = _add_channels_to_command(command, module.params['channels']) + command = _add_extras_to_command(command, module.params['extra_args']) + + rc, stdout, stderr = module.run_command(command) + + if rc != 0: + module.fail_json(msg='failed to update package ' + name) + + module.exit_json(changed=True, name=name, stdout=stdout, stderr=stderr) + + +def main(): + module = AnsibleModule( + argument_spec={ + 'name': {'required': True, 'type': 'str'}, + 'version': {'default': None, 'required': False, 'type': 'str'}, + 'state': { + 'default': 'present', + 'required': False, + 'choices': ['present', 'absent', 'latest'] + }, + 'channels': {'default': None, 'required': False}, + 'executable': {'default': None, 'required': False}, + 'extra_args': {'default': None, 'required': False, 'type': 'str'} + }, + supports_check_mode=True) + + conda = _find_conda(module, module.params['executable']) + name = module.params['name'] + state = module.params['state'] + version = module.params['version'] + + installed, installed_version = _check_installed(module, conda, name) + + if state == 'absent': + _remove_package(module, conda, installed, name) + elif state == 'present' or (state == 'latest' and not installed): + _install_package( + module, conda, installed, name, version, installed_version) + elif state == 'latest': + _update_package(module, conda, installed, name) + + +# import module snippets +from ansible.module_utils.basic import * + +if __name__ == '__main__': + main() diff --git a/ansible/roles/miniconda/tasks/main.yml b/ansible/roles/miniconda/tasks/main.yml new file mode 100644 index 0000000..57f03fa --- /dev/null +++ b/ansible/roles/miniconda/tasks/main.yml @@ -0,0 +1,12 @@ +--- +- name: download miniconda + get_url: + url: https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86.sh + dest: /tmp/miniconda.sh + mode: 0755 + +- name: install miniconda + command: bash /tmp/miniconda.sh -b -f + args: + creates: ~/miniconda3/bin/conda + become: no diff --git a/ansible/vagrant.yml b/ansible/vagrant.yml index 581af25..7f538cb 100644 --- a/ansible/vagrant.yml +++ b/ansible/vagrant.yml @@ -1,6 +1,9 @@ --- - hosts: all + roles: + - miniconda + tasks: - name: install deps @@ -9,14 +12,6 @@ state=present with_items: - git - - ansible - - python3 - - python3-dev - - python3-pip - - python-virtualenv - - libxml2-dev - - libxslt1-dev - - zlib1g-dev - lftp - pandoc - nodejs @@ -25,30 +20,25 @@ - mc - imagemagick - ack-grep - sudo: yes + become: yes - - name: clone batcave - git: - repo=https://github.com/xsteadfastx/batcave.git - dest=~/batcave - - - name: run batcave - command: - ansible-playbook -i hosts base.yml -c local - chdir=~/batcave + - name: install conda packages + conda: + name={{ item }} + executable=/home/vagrant/miniconda3/bin/conda + with_items: + - jupyter=1.0.0 - name: install blogging packages pip: name={{ item }} - executable=pip3 + executable=/home/vagrant/miniconda3/bin/pip with_items: - pelican==3.6.3 - markdown - - ipython[notebook]==2.4.1 - beautifulsoup4 - cookiecutter==1.2.1 - tzlocal - sudo: yes - name: clone pelican-plugins git: @@ -56,26 +46,6 @@ dest=/vagrant/pelican-plugins accept_hostkey=yes - - name: set timezone - copy: - content="Europe/Berlin" - dest=/etc/timezone - owner=root - group=root - mode=0644 - backup=yes - sudo: yes - notify: - - update timezone - - - name: export variables - lineinfile: - dest=~/.zshrc - line="{{ item }}" - with_items: - - export PYTHONIOENCODING=UTF-8 - - export LC_ALL=C.UTF-8 - - name: install js packages npm: name={{ item }} @@ -88,9 +58,49 @@ - gulp-less - gulp-minify-css + - name: set timezone + copy: + content="Europe/Berlin" + dest=/etc/timezone + owner=root + group=root + mode=0644 + backup=yes + become: yes + notify: + - update timezone + + - name: create ansible environment + command: /home/vagrant/miniconda3/bin/conda create -n ansible python=2.7 --yes + args: + creates: /home/vagrant/miniconda3/envs/ansible + + - name: install ansible + pip: + name=ansible + executable=/home/vagrant/miniconda3/envs/ansible/bin/pip + + - name: clone batcave + git: + repo=https://github.com/xsteadfastx/batcave.git + dest=~/batcave + + - name: run batcave + command: + /home/vagrant/miniconda3/envs/ansible/bin/ansible-playbook -i hosts base.yml -c local + chdir=~/batcave + + - name: export variables + lineinfile: + dest=~/.zshrc + line="{{ item }}" + with_items: + - export PYTHONIOENCODING=UTF-8 + - export LC_ALL=C.UTF-8 + handlers: - name: update timezone command: dpkg-reconfigure --frontend moninteractive tzdata - sudo: yes + become: yes diff --git a/vagrant/Dockerfile b/vagrant/Dockerfile index 6a8f57d..919e8d6 100644 --- a/vagrant/Dockerfile +++ b/vagrant/Dockerfile @@ -6,36 +6,25 @@ ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 RUN apt-get update \ - && apt-get install -y \ - software-properties-common \ - python-software-properties \ - - && add-apt-repository "deb http://archive.ubuntu.com/ubuntu trusty universe" \ - - && apt-get update \ - && apt-get install -y \ - openssh-server \ - supervisor + && apt-get install -y \ + openssh-server \ + && rm -rf /var/lib/apt/lists/* -RUN mkdir -p /var/run/sshd -RUN chmod 0755 /var/run/sshd +RUN mkdir -p /var/run/sshd \ + && chmod 0755 /var/run/sshd # Create and configure vagrant user RUN useradd --create-home -s /bin/bash vagrant -WORKDIR /home/vagrant # Configure SSH access -RUN mkdir -p /home/vagrant/.ssh -RUN echo "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key" > /home/vagrant/.ssh/authorized_keys -RUN chown -R vagrant: /home/vagrant/.ssh -RUN echo -n 'vagrant:vagrant' | chpasswd +RUN mkdir -p /home/vagrant/.ssh \ + && echo "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key" > /home/vagrant/.ssh/authorized_keys \ + && chown -R vagrant: /home/vagrant/.ssh \ + && echo -n 'vagrant:vagrant' | chpasswd # Enable passwordless sudo for the "vagrant" user -RUN echo 'vagrant ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/vagrant -RUN chmod 0440 /etc/sudoers.d/vagrant - -# Supervisor -COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf +RUN echo 'vagrant ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/vagrant \ + && chmod 0440 /etc/sudoers.d/vagrant EXPOSE 22 -CMD ["/usr/bin/supervisord"] +CMD ["/usr/sbin/sshd", "-D", "-o", "UseDNS=no", "-o", "UsePAM=no"]