| #!/bin/bash |
| # |
| # Setup the files and checkouts on a cluster telemetry machine. |
| # |
| |
| set -e |
| |
| # Install packages. |
| echo "Installing packages..." |
| sudo apt-get update |
| sudo apt-get -y install libosmesa-dev clang-3.6 poppler-utils netpbm \ |
| python3-django libgif-dev lua5.2 libnss3 python-setuptools python-pip \ |
| libglu1 libgtk3.0 xvfb gperf bison libglu1-mesa-dev libgbm-dev |
| |
| # Install Python packages. |
| cat <<EOF > /tmp/ct-requirements.txt |
| # |
| # This file is autogenerated by pip-compile with python 3.9 |
| # To update, run: |
| # |
| # pip-compile --generate-hashes requirements.in |
| # |
| crcmod==1.7 \ |
| --hash=sha256:dc7051a0db5f2bd48665a990d3ec1cc305a466a77358ca4492826f41f283601e |
| # via -r requirements.in |
| httplib2==0.20.4 \ |
| --hash=sha256:58a98e45b4b1a48273073f905d2961666ecf0fbac4250ea5b47aef259eb5c585 \ |
| --hash=sha256:8b6a905cb1c79eefd03f8669fd993c36dc341f7c558f056cb5a33b5c2f458543 |
| # via -r requirements.in |
| mock==4.0.3 \ |
| --hash=sha256:122fcb64ee37cfad5b3f48d7a7d51875d7031aaf3d8be7c42e2bee25044eee62 \ |
| --hash=sha256:7d3fbbde18228f4ff2f1f119a45cdffa458b4c0dee32eb4d2bb2f82554bac7bc |
| # via -r requirements.in |
| numpy==1.22.4 \ |
| --hash=sha256:0791fbd1e43bf74b3502133207e378901272f3c156c4df4954cad833b1380207 \ |
| --hash=sha256:1ce7ab2053e36c0a71e7a13a7475bd3b1f54750b4b433adc96313e127b870887 \ |
| --hash=sha256:2d487e06ecbf1dc2f18e7efce82ded4f705f4bd0cd02677ffccfb39e5c284c7e \ |
| --hash=sha256:37431a77ceb9307c28382c9773da9f306435135fae6b80b62a11c53cfedd8802 \ |
| --hash=sha256:3e1ffa4748168e1cc8d3cde93f006fe92b5421396221a02f2274aab6ac83b077 \ |
| --hash=sha256:425b390e4619f58d8526b3dcf656dde069133ae5c240229821f01b5f44ea07af \ |
| --hash=sha256:43a8ca7391b626b4c4fe20aefe79fec683279e31e7c79716863b4b25021e0e74 \ |
| --hash=sha256:4c6036521f11a731ce0648f10c18ae66d7143865f19f7299943c985cdc95afb5 \ |
| --hash=sha256:59d55e634968b8f77d3fd674a3cf0b96e85147cd6556ec64ade018f27e9479e1 \ |
| --hash=sha256:64f56fc53a2d18b1924abd15745e30d82a5782b2cab3429aceecc6875bd5add0 \ |
| --hash=sha256:7228ad13744f63575b3a972d7ee4fd61815b2879998e70930d4ccf9ec721dce0 \ |
| --hash=sha256:9ce7df0abeabe7fbd8ccbf343dc0db72f68549856b863ae3dd580255d009648e \ |
| --hash=sha256:a911e317e8c826ea632205e63ed8507e0dc877dcdc49744584dfc363df9ca08c \ |
| --hash=sha256:b89bf9b94b3d624e7bb480344e91f68c1c6c75f026ed6755955117de00917a7c \ |
| --hash=sha256:ba9ead61dfb5d971d77b6c131a9dbee62294a932bf6a356e48c75ae684e635b3 \ |
| --hash=sha256:c1d937820db6e43bec43e8d016b9b3165dcb42892ea9f106c70fb13d430ffe72 \ |
| --hash=sha256:cc7f00008eb7d3f2489fca6f334ec19ca63e31371be28fd5dad955b16ec285bd \ |
| --hash=sha256:d4c5d5eb2ec8da0b4f50c9a843393971f31f1d60be87e0fb0917a49133d257d6 \ |
| --hash=sha256:e96d7f3096a36c8754207ab89d4b3282ba7b49ea140e4973591852c77d09eb76 \ |
| --hash=sha256:f0725df166cf4785c0bc4cbfb320203182b1ecd30fee6e541c8752a92df6aa32 \ |
| --hash=sha256:f3eb268dbd5cfaffd9448113539e44e2dd1c5ca9ce25576f7c04a5453edc26fa \ |
| --hash=sha256:fb7a980c81dd932381f8228a426df8aeb70d59bbcda2af075b627bbc50207cba |
| # via |
| # -r requirements.in |
| # pandas |
| pandas==1.4.2 \ |
| --hash=sha256:0010771bd9223f7afe5f051eb47c4a49534345dfa144f2f5470b27189a4dd3b5 \ |
| --hash=sha256:061609334a8182ab500a90fe66d46f6f387de62d3a9cb9aa7e62e3146c712167 \ |
| --hash=sha256:09d8be7dd9e1c4c98224c4dfe8abd60d145d934e9fc1f5f411266308ae683e6a \ |
| --hash=sha256:295872bf1a09758aba199992c3ecde455f01caf32266d50abc1a073e828a7b9d \ |
| --hash=sha256:3228198333dd13c90b6434ddf61aa6d57deaca98cf7b654f4ad68a2db84f8cfe \ |
| --hash=sha256:385c52e85aaa8ea6a4c600a9b2821181a51f8be0aee3af6f2dcb41dafc4fc1d0 \ |
| --hash=sha256:51649ef604a945f781105a6d2ecf88db7da0f4868ac5d45c51cb66081c4d9c73 \ |
| --hash=sha256:5586cc95692564b441f4747c47c8a9746792e87b40a4680a2feb7794defb1ce3 \ |
| --hash=sha256:5a206afa84ed20e07603f50d22b5f0db3fb556486d8c2462d8bc364831a4b417 \ |
| --hash=sha256:5b79af3a69e5175c6fa7b4e046b21a646c8b74e92c6581a9d825687d92071b51 \ |
| --hash=sha256:5c54ea4ef3823108cd4ec7fb27ccba4c3a775e0f83e39c5e17f5094cb17748bc \ |
| --hash=sha256:8c5bf555b6b0075294b73965adaafb39cf71c312e38c5935c93d78f41c19828a \ |
| --hash=sha256:92bc1fc585f1463ca827b45535957815b7deb218c549b7c18402c322c7549a12 \ |
| --hash=sha256:95c1e422ced0199cf4a34385ff124b69412c4bc912011ce895582bee620dfcaa \ |
| --hash=sha256:b8134651258bce418cb79c71adeff0a44090c98d955f6953168ba16cc285d9f7 \ |
| --hash=sha256:be67c782c4f1b1f24c2f16a157e12c2693fd510f8df18e3287c77f33d124ed07 \ |
| --hash=sha256:c072c7f06b9242c855ed8021ff970c0e8f8b10b35e2640c657d2a541c5950f59 \ |
| --hash=sha256:d0d4f13e4be7ce89d7057a786023c461dd9370040bdb5efa0a7fe76b556867a0 \ |
| --hash=sha256:df82739e00bb6daf4bba4479a40f38c718b598a84654cbd8bb498fd6b0aa8c16 \ |
| --hash=sha256:f549097993744ff8c41b5e8f2f0d3cbfaabe89b4ae32c8c08ead6cc535b80139 \ |
| --hash=sha256:ff08a14ef21d94cdf18eef7c569d66f2e24e0bc89350bcd7d243dd804e3b5eb2 |
| # via -r requirements.in |
| psutil==5.9.1 \ |
| --hash=sha256:068935df39055bf27a29824b95c801c7a5130f118b806eee663cad28dca97685 \ |
| --hash=sha256:0904727e0b0a038830b019551cf3204dd48ef5c6868adc776e06e93d615fc5fc \ |
| --hash=sha256:0f15a19a05f39a09327345bc279c1ba4a8cfb0172cc0d3c7f7d16c813b2e7d36 \ |
| --hash=sha256:19f36c16012ba9cfc742604df189f2f28d2720e23ff7d1e81602dbe066be9fd1 \ |
| --hash=sha256:20b27771b077dcaa0de1de3ad52d22538fe101f9946d6dc7869e6f694f079329 \ |
| --hash=sha256:28976df6c64ddd6320d281128817f32c29b539a52bdae5e192537bc338a9ec81 \ |
| --hash=sha256:29a442e25fab1f4d05e2655bb1b8ab6887981838d22effa2396d584b740194de \ |
| --hash=sha256:3054e923204b8e9c23a55b23b6df73a8089ae1d075cb0bf711d3e9da1724ded4 \ |
| --hash=sha256:32c52611756096ae91f5d1499fe6c53b86f4a9ada147ee42db4991ba1520e574 \ |
| --hash=sha256:3a76ad658641172d9c6e593de6fe248ddde825b5866464c3b2ee26c35da9d237 \ |
| --hash=sha256:44d1826150d49ffd62035785a9e2c56afcea66e55b43b8b630d7706276e87f22 \ |
| --hash=sha256:4b6750a73a9c4a4e689490ccb862d53c7b976a2a35c4e1846d049dcc3f17d83b \ |
| --hash=sha256:56960b9e8edcca1456f8c86a196f0c3d8e3e361320071c93378d41445ffd28b0 \ |
| --hash=sha256:57f1819b5d9e95cdfb0c881a8a5b7d542ed0b7c522d575706a80bedc848c8954 \ |
| --hash=sha256:58678bbadae12e0db55186dc58f2888839228ac9f41cc7848853539b70490021 \ |
| --hash=sha256:645bd4f7bb5b8633803e0b6746ff1628724668681a434482546887d22c7a9537 \ |
| --hash=sha256:799759d809c31aab5fe4579e50addf84565e71c1dc9f1c31258f159ff70d3f87 \ |
| --hash=sha256:79c9108d9aa7fa6fba6e668b61b82facc067a6b81517cab34d07a84aa89f3df0 \ |
| --hash=sha256:91c7ff2a40c373d0cc9121d54bc5f31c4fa09c346528e6a08d1845bce5771ffc \ |
| --hash=sha256:9272167b5f5fbfe16945be3db475b3ce8d792386907e673a209da686176552af \ |
| --hash=sha256:944c4b4b82dc4a1b805329c980f270f170fdc9945464223f2ec8e57563139cf4 \ |
| --hash=sha256:a6a11e48cb93a5fa606306493f439b4aa7c56cb03fc9ace7f6bfa21aaf07c453 \ |
| --hash=sha256:a8746bfe4e8f659528c5c7e9af5090c5a7d252f32b2e859c584ef7d8efb1e689 \ |
| --hash=sha256:abd9246e4cdd5b554a2ddd97c157e292ac11ef3e7af25ac56b08b455c829dca8 \ |
| --hash=sha256:b14ee12da9338f5e5b3a3ef7ca58b3cba30f5b66f7662159762932e6d0b8f680 \ |
| --hash=sha256:b88f75005586131276634027f4219d06e0561292be8bd6bc7f2f00bdabd63c4e \ |
| --hash=sha256:c7be9d7f5b0d206f0bbc3794b8e16fb7dbc53ec9e40bbe8787c6f2d38efcf6c9 \ |
| --hash=sha256:d2d006286fbcb60f0b391741f520862e9b69f4019b4d738a2a45728c7e952f1b \ |
| --hash=sha256:db417f0865f90bdc07fa30e1aadc69b6f4cad7f86324b02aa842034efe8d8c4d \ |
| --hash=sha256:e7e10454cb1ab62cc6ce776e1c135a64045a11ec4c6d254d3f7689c16eb3efd2 \ |
| --hash=sha256:f65f9a46d984b8cd9b3750c2bdb419b2996895b005aefa6cbaba9a143b1ce2c5 \ |
| --hash=sha256:fea896b54f3a4ae6f790ac1d017101252c93f6fe075d0e7571543510f11d2676 |
| # via -r requirements.in |
| pyparsing==3.0.9 \ |
| --hash=sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb \ |
| --hash=sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc |
| # via httplib2 |
| python-dateutil==2.8.2 \ |
| --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ |
| --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 |
| # via pandas |
| pytz==2022.1 \ |
| --hash=sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7 \ |
| --hash=sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c |
| # via pandas |
| six==1.16.0 \ |
| --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ |
| --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 |
| # via python-dateutil |
| EOF |
| sudo pip install --require-hashes -r /tmp/ct-requirements.txt |
| |
| # Install openjdk-8. See skbug.com/6975 for context. |
| sudo apt-get -y install openjdk-8-jdk openjdk-8-jre |
| |
| echo "Installing Python..." |
| |
| sudo apt-get -y install autotools-dev blt-dev bzip2 dpkg-dev g++-multilib \ |
| gcc-multilib libbluetooth-dev libbz2-dev libexpat1-dev libffi-dev libffi6 \ |
| libffi6-dbg libgdbm-dev libgpm2 libncursesw5-dev libreadline-dev \ |
| libsqlite3-dev libssl-dev libtinfo-dev mime-support net-tools netbase \ |
| python-crypto python-mox3 python-pil python-ply quilt tk-dev zlib1g-dev \ |
| mesa-utils android-tools-adb python3-distutils |
| |
| # Install python3.8 (skbug.com/12021). This will not be needed when all of CT |
| # uses python3 and we can use it via CIPD instead. |
| sudo apt install python3.8 -y |
| sudo rm /usr/bin/python3 |
| sudo ln -s /usr/bin/python3.8 /usr/bin/python3 |
| |
| echo "Bring artifacts in from Google storage..." |
| # TODO(rmistry): Figure out which ones we really need. |
| /snap/bin/gsutil cp gs://cluster-telemetry-bucket/artifacts/bots/.gitconfig_ct ~/.gitconfig |
| /snap/bin/gsutil cp gs://cluster-telemetry-bucket/artifacts/bots/.netrc_ct ~/.netrc |
| /snap/bin/gsutil cp gs://cluster-telemetry-bucket/artifacts/bots/.boto_ct ~/.boto |
| |
| echo "Checking out depot_tools..." |
| if [ ! -d ~/depot_tools ]; then |
| cd ~ |
| git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git |
| echo 'export PATH=~/depot_tools:$PATH' >> ~/.bashrc |
| fi |
| PATH=$PATH:~/depot_tools |
| |
| # To keep .boto file consistent with golo bots. |
| cp ~/.boto ~/.boto.puppet-bak |
| |
| # CT uses this directory for storage of artifacts. |
| mkdir -p /b/storage |
| |
| # If the bot is a builder then checkout Chromium and Skia repositories. |
| if [[ $(hostname -s) = ct-*-builder* ]]; then |
| echo "Checking out Chromium repository..." |
| mkdir -p /b/storage/chromium |
| cd /b/storage/chromium |
| if [[ $(hostname -s) = *android* ]]; then |
| # Say yes to prompts for installing Android SDK. |
| yes | ~/depot_tools/fetch android |
| else |
| ~/depot_tools/fetch chromium |
| fi |
| cd src |
| git checkout master |
| ~/depot_tools/gclient sync |
| |
| echo "Checking out Skia repository..." |
| mkdir /b/skia-repo/ |
| cd /b/skia-repo/ |
| cat > .gclient << EOF |
| solutions = [ |
| { 'name' : 'trunk', |
| 'url' : 'https://skia.googlesource.com/skia.git', |
| 'deps_file' : 'DEPS', |
| 'managed' : True, |
| 'custom_deps' : { |
| }, |
| 'safesync_url': '', |
| }, |
| ] |
| EOF |
| ~/depot_tools/gclient sync |
| cd trunk |
| git checkout master |
| fi |
| |
| # Get access token from metadata. |
| TOKEN_URL="http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token" |
| TOKEN="$(curl "${TOKEN_URL}" --header "Metadata-Flavor: Google" | \ |
| python -c "import sys, json; print json.load(sys.stdin)['access_token']")" |
| # Bootstrap Swarming. |
| mkdir -p /b/s |
| SWARMING=https://chrome-swarming.appspot.com |
| HOSTNAME=`hostname` |
| curl "${SWARMING}/bot_code?bot_id=${HOSTNAME}" --header "Authorization":"Bearer $TOKEN" \ |
| --location --output /b/s/swarming_bot.zip |
| |
| # See skbug.com/9425 for why LimitNOFILE is set. |
| cat <<EOF | sudo tee /etc/systemd/system/swarming_bot.service |
| [Unit] |
| Description=Swarming bot |
| After=network.target |
| |
| [Service] |
| Type=simple |
| LimitNOFILE=50000 |
| User=chrome-bot |
| Restart=on-failure |
| RestartSec=10 |
| ExecStart=/usr/bin/env python3 /b/s/swarming_bot.zip start_bot |
| |
| [Install] |
| WantedBy=default.target |
| EOF |
| |
| sudo systemctl daemon-reload |
| sudo systemctl enable swarming_bot.service |
| |
| echo |
| echo "The setup script has completed!" |
| echo |