[skolo] Provide our own Python to power Ansible on Macs.
macOS 13 quit shipping Python (shipping only an XCode shim), so we have
no choice. However, I found a way to embed the Python bootstrapping in
the main playbook, so we can bring our own Python without having a
separate setup step. Just run mac.yml on a brand new unboxed machine,
and away you go.
Also discovered changed_when, which gets rid of spurious yellow status
messages.
Change-Id: If50374b73599b679f79634d28b213171f62382c1
Reviewed-on: https://skia-review.googlesource.com/c/buildbot/+/564918
Commit-Queue: Erik Rose <erikrose@google.com>
Reviewed-by: Joe Gregorio <jcgregorio@google.com>
diff --git a/skolo/ansible/hosts.yml b/skolo/ansible/hosts.yml
index 658225b..172233d 100644
--- a/skolo/ansible/hosts.yml
+++ b/skolo/ansible/hosts.yml
@@ -296,72 +296,14 @@
vars:
install_test_machine_monitor__start_swarming: true
skolo_group: staff
-
- # Use the Python which actually ships with the Mac. This is Python 2 most of
- # the time, which is still supported module-side as of Ansible 2.12.
- # /usr/bin/python3 is present but often an XCode shim, which launches
- # xcode-select, which requires GUI confirmation to proceed. The alternative
- # is to parse the CLI output `softwareupdate -l`, whose format varies
- # between OS releases. (See
- # https://apple.stackexchange.com/questions/107307/how-can-i-install-the
- # -command-line-tools-completely-from-the-command-line.) Or to re-introduce
- # a separate pre-Ansible setup script, which I'd like to avoid.
- #
- # Which of {python, python3} are truly present and which are shims vary from
- # macOS version to macOS version. Ansible 2.10 worked fine, but 2.12 broke
- # interpreter discovery, even with interpreter_python=auto_legacy. So we use
- # a manual lookup table. See b/233103849 for more detail.
- #
- # Here are the macOS versions where /usr/bin/python is the real, non-shim Python:
- macs_with_usr_bin_python:
- children:
- macos_10_12:
- hosts:
- skia-e-mac-[101:102]:
- macos_10_13:
- hosts:
- build[19:20]-m3:
- build[337:341]-a9:
- skia-e-mac-[101:104]:
- skia-e-mac-[233:234]:
- skia-e-mac-236:
- skia-e-mac-[333:335]:
- v-mac-[155:156]-g576:
- macos_10_14:
- hosts:
- skia-e-mac-[105:106]:
- v-mac-[157:158]-g576:
- macos_10_15:
- hosts:
- build336-a9:
- mac-[140:145]-h525:
- mac-[653:665]-g580:
- skia-e-mac-[107:108]:
- skia-e-mac-[230:232]:
- skia-e-mac-[330:332]:
- macos_11:
- hosts:
- skia-e-mac-235:
- skia-e-mac-[237:238]:
- skia-e-mac-[336:338]:
- # Apple started shipping python3 (and omitting python 2) in 12.3 exactly.
- macos_12_1:
- hosts:
- skia-e-mac-[109:113]:
- skia-e-mac-[116:117]:
- skia-e-mac-[239:240]:
- skia-e-mac-[339:340]:
- vars:
- ansible_python_interpreter: /usr/bin/python
-
- # Here are the macOS versions where /usr/bin/python3 is the real, non-shim Python:
- macs_with_usr_bin_python3:
- children:
- macos_12_3:
- hosts:
- skia-e-mac-[114:115]:
- vars:
- ansible_python_interpreter: /usr/bin/python3
+ # Always use the Python we provide ourselves. Python shipped with the
+ # Mac until macOS 13, but the version varied, and some copies were
+ # actually decoys--not real Python interpreters but shims which launched
+ # xcode-select, which itself required GUI confirmation to proceed. In
+ # addition, Ansible 2.12 broke interpreter discovery (2.10 worked fine),
+ # even with interpreter_python=auto_legacy. So we keep it consistent and
+ # use the one we stick on the box. See b/233103849 for more detail.
+ ansible_python_interpreter: /usr/local/bin/python3
all_win:
children:
diff --git a/skolo/ansible/switchboard/mac.yml b/skolo/ansible/switchboard/mac.yml
index a119145..b398493 100644
--- a/skolo/ansible/switchboard/mac.yml
+++ b/skolo/ansible/switchboard/mac.yml
@@ -1,3 +1,7 @@
+- name: Ensure Python is around for Ansible to use.
+ # It stopped being included in macOS 13.
+ import_playbook: prepare-mac-for-ansible.yml
+
- hosts: all_mac
user: chrome-bot
gather_facts: true
diff --git a/skolo/ansible/switchboard/prepare-mac-for-ansible.yml b/skolo/ansible/switchboard/prepare-mac-for-ansible.yml
new file mode 100644
index 0000000..764ba53
--- /dev/null
+++ b/skolo/ansible/switchboard/prepare-mac-for-ansible.yml
@@ -0,0 +1,48 @@
+# Install Python 3 so we can do more than issue 'raw' commands.
+#
+# You don't need to run this by hand; mac.yml will run it as necessary.
+- hosts: all_mac
+ gather_facts: false
+
+ pre_tasks:
+ - name: See if our version of Python is already installed.
+ raw: /usr/local/bin/python3 --version
+ register: local_python_version
+ failed_when: false # Tolerate the binary not being there.
+ changed_when: false
+
+ - name: Install Python 3 and prerequisites.
+ when: local_python_version.stdout_lines[0] != "Python 3.9.6"
+ block:
+ - name: Check for Rosetta.
+ raw: test -e /Library/Apple/usr/libexec/oah/libRosettaRuntime
+ register: rosetta_present
+ failed_when: false
+ changed_when: false
+
+ - name: Check architecture.
+ raw: arch
+ register: cpu_arch
+ changed_when: false
+
+ - name: Install Rosetta on ARM machines if it's absent.
+ # The Python installer package isn't marked as being ARM-native, so it
+ # refuses to run unless Rosetta 2 is around. This is true up to and
+ # including the 3.10.1 universal2 installer.
+ become: true
+ when: rosetta_present.rc != 0 and cpu_arch.stdout_lines[0] == "arm64"
+ raw: softwareupdate --install-rosetta --agree-to-license
+
+ - name: Install Python 3.
+ vars:
+ # Contains universal binaries. Happily runs on macOS 10.9+, despite
+ # name.
+ python_pkg: python-3.9.6-macos11.pkg
+ block:
+ - name: Download Python installer.
+ # Works even if the file is already there:
+ raw: curl -O 'https://www.python.org/ftp/python/3.9.6/{{ python_pkg }}' --output-dir ~{{ skolo_account }}/Downloads
+
+ - name: Run installer.
+ become: true
+ raw: /usr/sbin/installer -verboseR -package ~{{ skolo_account }}/Downloads/{{ python_pkg }} -target /
diff --git a/skolo/ansible/switchboard/roles/swarming_needs/tasks/mac.yml b/skolo/ansible/switchboard/roles/swarming_needs/tasks/mac.yml
index d44ba52..f9e141e 100644
--- a/skolo/ansible/switchboard/roles/swarming_needs/tasks/mac.yml
+++ b/skolo/ansible/switchboard/roles/swarming_needs/tasks/mac.yml
@@ -18,34 +18,7 @@
owner: '{{ skolo_account }}'
mode: 0755
-- name: Install Rosetta on ARM machines if it's absent.
- # The Python installer package isn't marked as being ARM-native, so it refuses
- # to run unless Rosetta 2 is around. This is true as of the 3.10.1 universal2
- # installer.
- command:
- cmd: softwareupdate --install-rosetta --agree-to-license
- creates: /Library/Apple/usr/libexec/oah/libRosettaRuntime
- when: ansible_facts['machine'] == 'arm64'
-
-- name: Install Python 3.
- vars:
- # Contains universal binaries. Happily runs on macOS 10.9+, despite name.
- python_pkg: python-3.9.6-macos11.pkg
- block:
- - name: Download Python installer.
- # Do this with curl instead of the get_url module because 10.12's Python
- # has too old a version of TLS to work.
- command:
- cmd: curl -O 'https://www.python.org/ftp/python/3.9.6/{{ python_pkg }}'
- chdir: ~{{ skolo_account }}/Downloads
- creates: ~{{ skolo_account }}/Downloads/{{ python_pkg }}
- warn: false
-
- - name: Run installer.
- become: true
- command:
- cmd: /usr/sbin/installer -verboseR -package ~{{ skolo_account }}/Downloads/{{ python_pkg }} -target /
- creates: /usr/local/bin/python3
+# Python 3.9.6 is taken to be already present, since it's installed by prepare-mac-for-ansible.yml.
- name: Install pyobjc.
block:
